[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [tor/master] Split geoip from geoip-related stats.

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [tor/master] Split geoip from geoip-related stats.
From: nickm@xxxxxxxxxxxxxx
Date: Thu, 27 Sep 2018 20:30:06 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Thu, 27 Sep 2018 16:30:28 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: Nick Mathewson <nickm@xxxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>
commit f403af22076b0092807822ddc9890acdbd62f578
Author: Nick Mathewson <nickm@xxxxxxxxxxxxxx>
Date:   Thu Sep 27 09:35:47 2018 -0400

    Split geoip from geoip-related stats.
    
    This commit just moves the code to two separate files. The geoip
    code still has a few needless dependencies on core/* and features/*.
---
 src/app/config/config.c             |    1 +
 src/app/main/main.c                 |    3 +
 src/core/include.am                 |    4 +
 src/core/mainloop/mainloop.c        |    2 +-
 src/core/or/channel.c               |   23 +-
 src/core/or/dos.c                   |   10 +-
 src/core/or/relay.c                 |    2 +-
 src/core/or/status.c                |    2 +-
 src/feature/control/control.c       |    3 +-
 src/feature/control/getinfo_geoip.c |   45 ++
 src/feature/control/getinfo_geoip.h |   14 +
 src/feature/dircache/dircache.c     |    2 +-
 src/feature/dircommon/directory.c   |    2 +-
 src/feature/relay/router.c          |    1 +
 src/feature/stats/geoip.c           | 1433 +----------------------------------
 src/feature/stats/geoip.h           |  129 +---
 src/feature/stats/geoip_stats.c     | 1422 ++++++++++++++++++++++++++++++++++
 src/feature/stats/geoip_stats.h     |  139 ++++
 src/test/test_dir_handle_get.c      |    1 +
 src/test/test_dos.c                 |    3 +-
 src/test/test_geoip.c               |    1 +
 21 files changed, 1673 insertions(+), 1569 deletions(-)

diff --git a/src/app/config/config.c b/src/app/config/config.c
index 2818cdaad..0909ced50 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -103,6 +103,7 @@
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
diff --git a/src/app/main/main.c b/src/app/main/main.c
index c756e55b9..2266a5ff7 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -27,6 +27,7 @@
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
+#include "core/or/dos.h"
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "core/or/relay.h"
@@ -61,6 +62,7 @@
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
@@ -755,6 +757,7 @@ tor_free_all(int postfork)
     evdns_shutdown(1);
   }
   geoip_free_all();
+  geoip_stats_free_all();
   dirvote_free_all();
   routerlist_free_all();
   networkstatus_free_all();
diff --git a/src/core/include.am b/src/core/include.am
index 68bbf9474..1a26a65c2 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -60,6 +60,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/client/transports.c		\
 	src/feature/control/control.c		\
 	src/feature/control/fmt_serverstatus.c  \
+	src/feature/control/getinfo_geoip.c	\
 	src/feature/dirauth/keypin.c		\
 	src/feature/dircache/conscache.c	\
 	src/feature/dircache/consdiffmgr.c	\
@@ -117,6 +118,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/rend/rendmid.c		\
 	src/feature/rend/rendservice.c		\
 	src/feature/stats/geoip.c		\
+	src/feature/stats/geoip_stats.c		\
 	src/feature/stats/rephist.c		\
 	src/feature/stats/predict_ports.c
 
@@ -254,6 +256,7 @@ noinst_HEADERS +=					\
 	src/feature/control/control.h			\
 	src/feature/control/control_connection_st.h	\
 	src/feature/control/fmt_serverstatus.h		\
+	src/feature/control/getinfo_geoip.h		\
 	src/feature/dirauth/authmode.h			\
 	src/feature/dirauth/bwauth.h			\
 	src/feature/dirauth/dircollate.h		\
@@ -348,6 +351,7 @@ noinst_HEADERS +=					\
 	src/feature/rend/rendmid.h			\
 	src/feature/rend/rendservice.h			\
 	src/feature/stats/geoip.h			\
+	src/feature/stats/geoip_stats.h			\
 	src/feature/stats/rephist.h			\
 	src/feature/stats/predict_ports.h
 
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index ebfdf1c94..4eaeef46c 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -92,7 +92,7 @@
 #include "feature/relay/selftest.h"
 #include "feature/rend/rendcache.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/container/buffers.h"
diff --git a/src/core/or/channel.c b/src/core/or/channel.c
index e9011520a..fe3785d52 100644
--- a/src/core/or/channel.c
+++ b/src/core/or/channel.c
@@ -58,28 +58,29 @@
 #define CHANNEL_PRIVATE_
 
 #include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/mainloop.h"
 #include "core/or/channel.h"
-#include "core/or/channeltls.h"
 #include "core/or/channelpadding.h"
+#include "core/or/channeltls.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
+#include "core/or/circuitmux.h"
 #include "core/or/circuitstats.h"
-#include "app/config/config.h"
 #include "core/or/connection_or.h" /* For var_cell_free() */
-#include "core/or/circuitmux.h"
-#include "feature/client/entrynodes.h"
-#include "feature/stats/geoip.h"
-#include "core/mainloop/mainloop.h"
-#include "feature/nodelist/nodelist.h"
+#include "core/or/dos.h"
 #include "core/or/relay.h"
-#include "feature/stats/rephist.h"
-#include "feature/relay/router.h"
-#include "feature/nodelist/routerlist.h"
 #include "core/or/scheduler.h"
-#include "lib/time/compat_time.h"
+#include "feature/client/entrynodes.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
 #include "feature/rend/rendservice.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
 #include "lib/evloop/timers.h"
+#include "lib/time/compat_time.h"
 
 #include "core/or/cell_queue_st.h"
 
diff --git a/src/core/or/dos.c b/src/core/or/dos.c
index c59e2f636..4303b8fd5 100644
--- a/src/core/or/dos.c
+++ b/src/core/or/dos.c
@@ -9,17 +9,17 @@
 #define DOS_PRIVATE
 
 #include "core/or/or.h"
-#include "core/or/channel.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
-#include "core/or/connection_or.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/stats/geoip.h"
 #include "core/mainloop/mainloop.h"
+#include "core/or/channel.h"
+#include "core/or/connection_or.h"
+#include "core/or/relay.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
-#include "core/or/relay.h"
 #include "feature/relay/routermode.h"
+#include "feature/stats/geoip_stats.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
 #include "core/or/dos.h"
 
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index 88bcdf070..9e45a3524 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -65,7 +65,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircommon/directory.h"
 #include "feature/relay/dns.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/hs/hs_cache.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
diff --git a/src/core/or/status.c b/src/core/or/status.c
index d8a9da0a5..cdb6df123 100644
--- a/src/core/or/status.c
+++ b/src/core/or/status.c
@@ -30,13 +30,13 @@
 #include "feature/hs/hs_stats.h"
 #include "feature/hs/hs_service.h"
 #include "core/or/dos.h"
+#include "feature/stats/geoip_stats.h"
 
 #include "app/config/or_state_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "lib/tls/tortls.h"
 
 static void log_accounting(const time_t now, const or_options_t *options);
-#include "feature/stats/geoip.h"
 
 /** Return the total number of circuits. */
 STATIC int
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index b8ddb3c13..d02ecbac8 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -60,6 +60,7 @@
 #include "feature/client/entrynodes.h"
 #include "feature/control/control.h"
 #include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dirclient/dlstatus.h"
@@ -83,7 +84,7 @@
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "lib/container/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c
new file mode 100644
index 000000000..35eb3f6ae
--- /dev/null
+++ b/src/feature/control/getinfo_geoip.c
@@ -0,0 +1,45 @@
+
+#include "core/or/or.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/stats/geoip.h"
+
+/** Helper used to implement GETINFO ip-to-country/... controller command. */
+int
+getinfo_helper_geoip(control_connection_t *control_conn,
+                     const char *question, char **answer,
+                     const char **errmsg)
+{
+  (void)control_conn;
+  if (!strcmpstart(question, "ip-to-country/")) {
+    int c;
+    sa_family_t family;
+    tor_addr_t addr;
+    question += strlen("ip-to-country/");
+
+    if (!strcmp(question, "ipv4-available") ||
+        !strcmp(question, "ipv6-available")) {
+      family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
+      const int available = geoip_is_loaded(family);
+      tor_asprintf(answer, "%d", !! available);
+      return 0;
+    }
+
+    family = tor_addr_parse(&addr, question);
+    if (family != AF_INET && family != AF_INET6) {
+      *errmsg = "Invalid address family";
+      return -1;
+    }
+    if (!geoip_is_loaded(family)) {
+      *errmsg = "GeoIP data not loaded";
+      return -1;
+    }
+    if (family == AF_INET)
+      c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
+    else                      /* AF_INET6 */
+      c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
+    *answer = tor_strdup(geoip_get_country_name(c));
+  }
+  return 0;
+}
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
new file mode 100644
index 000000000..ff77cefec
--- /dev/null
+++ b/src/feature/control/getinfo_geoip.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_GETINFO_GEOIP_H
+#define TOR_GETINFO_GEOIP_H
+
+int getinfo_helper_geoip(control_connection_t *control_conn,
+                     const char *question, char **answer,
+                     const char **errmsg);
+
+#endif
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index 76bde0b78..872a88018 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -25,7 +25,7 @@
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
 
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
index 8116d72c6..dff68fcf0 100644
--- a/src/feature/dircommon/directory.c
+++ b/src/feature/dircommon/directory.c
@@ -12,7 +12,7 @@
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dircommon/fp_pair.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
 #include "feature/dircommon/dir_connection_st.h"
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 8629ad956..494634845 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -37,6 +37,7 @@
 #include "feature/relay/routermode.h"
 #include "feature/relay/selftest.h"
 #include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_format.h"
diff --git a/src/feature/stats/geoip.c b/src/feature/stats/geoip.c
index 7bae44848..885f26bc3 100644
--- a/src/feature/stats/geoip.c
+++ b/src/feature/stats/geoip.c
@@ -31,12 +31,9 @@
 #include "core/or/or.h"
 #include "ht.h"
 #include "lib/container/buffers.h"
-#include "app/config/config.h"
-#include "feature/control/control.h"
-#include "feature/client/dnsserv.h"
-#include "core/or/dos.h"
+#include "app/config/config.h" // XXXX
 #include "feature/stats/geoip.h"
-#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerlist.h" // XXXX
 
 #include "lib/container/order.h"
 #include "lib/time/tvdiff.h"
@@ -57,11 +54,6 @@ typedef struct geoip_ipv6_entry_t {
   intptr_t country; /**< An index into geoip_countries */
 } geoip_ipv6_entry_t;
 
-/** A per-country record for GeoIP request history. */
-typedef struct geoip_country_t {
-  char countrycode[3];
-} geoip_country_t;
-
 /** A list of geoip_country_t */
 static smartlist_t *geoip_countries = NULL;
 /** A map from lowercased country codes to their position in geoip_countries.
@@ -76,68 +68,14 @@ static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
 static char geoip_digest[DIGEST_LEN];
 static char geoip6_digest[DIGEST_LEN];
 
-/** Number of entries in n_v3_ns_requests */
-size_t n_v3_ns_requests_len = 0;
-/** Array, indexed by country index, of number of v3 networkstatus requests
- * received from that country */
-static uint32_t *n_v3_ns_requests;
-
-/* Total size in bytes of the geoip client history cache. Used by the OOM
- * handler. */
-static size_t geoip_client_history_cache_size;
-
-/* Increment the geoip client history cache size counter with the given bytes.
- * This prevents an overflow and set it to its maximum in that case. */
-static inline void
-geoip_increment_client_history_cache_size(size_t bytes)
+/** Return a list of geoip_country_t for all known countries. */
+const smartlist_t *
+geoip_get_countries(void)
 {
-  /* This is shockingly high, lets log it so it can be reported. */
-  IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
-    geoip_client_history_cache_size = SIZE_MAX;
-    return;
-  }
-  geoip_client_history_cache_size += bytes;
-}
-
-/* Decrement the geoip client history cache size counter with the given bytes.
- * This prevents an underflow and set it to 0 in that case. */
-static inline void
-geoip_decrement_client_history_cache_size(size_t bytes)
-{
-  /* Going below 0 means that we either allocated an entry without
-   * incrementing the counter or we have different sizes when allocating and
-   * freeing. It shouldn't happened so log it. */
-  IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
-    geoip_client_history_cache_size = 0;
-    return;
-  }
-  geoip_client_history_cache_size -= bytes;
-}
-
-/** Add 1 to the count of v3 ns requests received from <b>country</b>. */
-static void
-increment_v3_ns_request(country_t country)
-{
-  if (country < 0)
-    return;
-
-  if ((size_t)country >= n_v3_ns_requests_len) {
-    /* We need to reallocate the array. */
-    size_t new_len;
-    if (n_v3_ns_requests_len == 0)
-      new_len = 256;
-    else
-      new_len = n_v3_ns_requests_len * 2;
-    if (new_len <= (size_t)country)
-      new_len = ((size_t)country)+1;
-    n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
-                                        sizeof(uint32_t));
-    memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
-           sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
-    n_v3_ns_requests_len = new_len;
+  if (geoip_countries == NULL) {
+    init_geoip_countries();
   }
-
-  n_v3_ns_requests[country] += 1;
+  return geoip_countries;
 }
 
 /** Return the index of the <b>country</b>'s entry in the GeoIP
@@ -330,14 +268,6 @@ geoip_ipv6_compare_key_to_entry_(const void *_key, const void **_member)
     return 0;
 }
 
-/** Return 1 if we should collect geoip stats on bridge users, and
- * include them in our extrainfo descriptor. Else return 0. */
-int
-should_record_bridge_info(const or_options_t *options)
-{
-  return options->BridgeRelay && options->BridgeRecordUsageByCountry;
-}
-
 /** Set up a new list of geoip countries with no countries (yet) set in it,
  * except for the unknown country.
  */
@@ -448,7 +378,7 @@ geoip_load_file(sa_family_t family, const char *filename)
  * be less than geoip_get_n_countries().  To decode it, call
  * geoip_get_country_name().
  */
-STATIC int
+int
 geoip_get_country_by_ipv4(uint32_t ipaddr)
 {
   geoip_ipv4_entry_t *ent;
@@ -464,7 +394,7 @@ geoip_get_country_by_ipv4(uint32_t ipaddr)
  * 0 for the 'unknown country'.  The return value will always be less than
  * geoip_get_n_countries().  To decode it, call geoip_get_country_name().
  */
-STATIC int
+int
 geoip_get_country_by_ipv6(const struct in6_addr *addr)
 {
   geoip_ipv6_entry_t *ent;
@@ -540,1328 +470,6 @@ geoip_db_digest(sa_family_t family)
     return hex_str(geoip6_digest, DIGEST_LEN);
 }
 
-/** Largest allowable value for last_seen_in_minutes.  (It's a 30-bit field,
- * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
- */
-#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
-
-/** Map from client IP address to last time seen. */
-static HT_HEAD(clientmap, clientmap_entry_t) client_history =
-     HT_INITIALIZER();
-
-/** Hashtable helper: compute a hash of a clientmap_entry_t. */
-static inline unsigned
-clientmap_entry_hash(const clientmap_entry_t *a)
-{
-  unsigned h = (unsigned) tor_addr_hash(&a->addr);
-
-  if (a->transport_name)
-    h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
-
-  return h;
-}
-/** Hashtable helper: compare two clientmap_entry_t values for equality. */
-static inline int
-clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
-{
-  if (strcmp_opt(a->transport_name, b->transport_name))
-    return 0;
-
-  return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
-         a->action == b->action;
-}
-
-HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq)
-HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
-
-#define clientmap_entry_free(ent) \
-  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
-
-/** Return the size of a client map entry. */
-static inline size_t
-clientmap_entry_size(const clientmap_entry_t *ent)
-{
-  tor_assert(ent);
-  return (sizeof(clientmap_entry_t) +
-          (ent->transport_name ? strlen(ent->transport_name) : 0));
-}
-
-/** Free all storage held by <b>ent</b>. */
-static void
-clientmap_entry_free_(clientmap_entry_t *ent)
-{
-  if (!ent)
-    return;
-
-  /* This entry is about to be freed so pass it to the DoS subsystem to see if
-   * any actions can be taken about it. */
-  dos_geoip_entry_about_to_free(ent);
-  geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
-
-  tor_free(ent->transport_name);
-  tor_free(ent);
-}
-
-/* Return a newly allocated clientmap entry with the given action and address
- * that are mandatory. The transport_name can be optional. This can't fail. */
-static clientmap_entry_t *
-clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
-                    const char *transport_name)
-{
-  clientmap_entry_t *entry;
-
-  tor_assert(action == GEOIP_CLIENT_CONNECT ||
-             action == GEOIP_CLIENT_NETWORKSTATUS);
-  tor_assert(addr);
-
-  entry = tor_malloc_zero(sizeof(clientmap_entry_t));
-  entry->action = action;
-  tor_addr_copy(&entry->addr, addr);
-  if (transport_name) {
-    entry->transport_name = tor_strdup(transport_name);
-  }
-
-  /* Allocated and initialized, note down its size for the OOM handler. */
-  geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
-
-  return entry;
-}
-
-/** Clear history of connecting clients used by entry and bridge stats. */
-static void
-client_history_clear(void)
-{
-  clientmap_entry_t **ent, **next, *this;
-  for (ent = HT_START(clientmap, &client_history); ent != NULL;
-       ent = next) {
-    if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
-      this = *ent;
-      next = HT_NEXT_RMV(clientmap, &client_history, ent);
-      clientmap_entry_free(this);
-    } else {
-      next = HT_NEXT(clientmap, &client_history, ent);
-    }
-  }
-}
-
-/** Note that we've seen a client connect from the IP <b>addr</b>
- * at time <b>now</b>. Ignored by all but bridges and directories if
- * configured accordingly. */
-void
-geoip_note_client_seen(geoip_client_action_t action,
-                       const tor_addr_t *addr,
-                       const char *transport_name,
-                       time_t now)
-{
-  const or_options_t *options = get_options();
-  clientmap_entry_t *ent;
-
-  if (action == GEOIP_CLIENT_CONNECT) {
-    /* Only remember statistics if the DoS mitigation subsystem is enabled. If
-     * not, only if as entry guard or as bridge. */
-    if (!dos_enabled()) {
-      if (!options->EntryStatistics && !should_record_bridge_info(options)) {
-        return;
-      }
-    }
-  } else {
-    /* Only gather directory-request statistics if configured, and
-     * forcibly disable them on bridge authorities. */
-    if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
-      return;
-  }
-
-  log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
-            safe_str_client(fmt_addr((addr))),
-            transport_name ? transport_name : "<no transport>");
-
-  ent = geoip_lookup_client(addr, transport_name, action);
-  if (! ent) {
-    ent = clientmap_entry_new(action, addr, transport_name);
-    HT_INSERT(clientmap, &client_history, ent);
-  }
-  if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
-    ent->last_seen_in_minutes = (unsigned)(now/60);
-  else
-    ent->last_seen_in_minutes = 0;
-
-  if (action == GEOIP_CLIENT_NETWORKSTATUS) {
-    int country_idx = geoip_get_country_by_addr(addr);
-    if (country_idx < 0)
-      country_idx = 0; /** unresolved requests are stored at index 0. */
-    increment_v3_ns_request(country_idx);
-  }
-}
-
-/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
- * older than a certain time. */
-static int
-remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
-{
-  time_t cutoff = *(time_t*)_cutoff / 60;
-  if (ent->last_seen_in_minutes < cutoff) {
-    clientmap_entry_free(ent);
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-/** Forget about all clients that haven't connected since <b>cutoff</b>. */
-void
-geoip_remove_old_clients(time_t cutoff)
-{
-  clientmap_HT_FOREACH_FN(&client_history,
-                          remove_old_client_helper_,
-                          &cutoff);
-}
-
-/* Return a client entry object matching the given address, transport name and
- * geoip action from the clientmap. NULL if not found. The transport_name can
- * be NULL. */
-clientmap_entry_t *
-geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
-                    geoip_client_action_t action)
-{
-  clientmap_entry_t lookup;
-
-  tor_assert(addr);
-
-  /* We always look for a client connection with no transport. */
-  tor_addr_copy(&lookup.addr, addr);
-  lookup.action = action;
-  lookup.transport_name = (char *) transport_name;
-
-  return HT_FIND(clientmap, &client_history, &lookup);
-}
-
-/* Cleanup client entries older than the cutoff. Used for the OOM. Return the
- * number of bytes freed. If 0 is returned, nothing was freed. */
-static size_t
-oom_clean_client_entries(time_t cutoff)
-{
-  size_t bytes = 0;
-  clientmap_entry_t **ent, **ent_next;
-
-  for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
-    clientmap_entry_t *entry = *ent;
-    if (entry->last_seen_in_minutes < (cutoff / 60)) {
-      ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
-      bytes += clientmap_entry_size(entry);
-      clientmap_entry_free(entry);
-    } else {
-      ent_next = HT_NEXT(clientmap, &client_history, ent);
-    }
-  }
-  return bytes;
-}
-
-/* Below this minimum lifetime, the OOM won't cleanup any entries. */
-#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
-/* The OOM moves the cutoff by that much every run. */
-#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
-
-/* Cleanup the geoip client history cache called from the OOM handler. Return
- * the amount of bytes removed. This can return a value below or above
- * min_remove_bytes but will stop as oon as the min_remove_bytes has been
- * reached. */
-size_t
-geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
-{
-  time_t k;
-  size_t bytes_removed = 0;
-
-  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
-  tor_assert(min_remove_bytes != 0);
-
-  /* Set k to the initial cutoff of an entry. We then going to move it by step
-   * to try to remove as much as we can. */
-  k = WRITE_STATS_INTERVAL;
-
-  do {
-    time_t cutoff;
-
-    /* If k has reached the minimum lifetime, we have to stop else we might
-     * remove every single entries which would be pretty bad for the DoS
-     * mitigation subsystem if by just filling the geoip cache, it was enough
-     * to trigger the OOM and clean every single entries. */
-    if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
-      break;
-    }
-
-    cutoff = now - k;
-    bytes_removed += oom_clean_client_entries(cutoff);
-    k -= GEOIP_CLIENT_CACHE_OOM_STEP;
-  } while (bytes_removed < min_remove_bytes);
-
-  return bytes_removed;
-}
-
-/* Return the total size in bytes of the client history cache. */
-size_t
-geoip_client_cache_total_allocation(void)
-{
-  return geoip_client_history_cache_size;
-}
-
-/** How many responses are we giving to clients requesting v3 network
- * statuses? */
-static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
-
-/** Note that we've rejected a client's request for a v3 network status
- * for reason <b>reason</b> at time <b>now</b>. */
-void
-geoip_note_ns_response(geoip_ns_response_t response)
-{
-  static int arrays_initialized = 0;
-  if (!get_options()->DirReqStatistics)
-    return;
-  if (!arrays_initialized) {
-    memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
-    arrays_initialized = 1;
-  }
-  tor_assert(response < GEOIP_NS_RESPONSE_NUM);
-  ns_v3_responses[response]++;
-}
-
-/** Do not mention any country from which fewer than this number of IPs have
- * connected.  This conceivably avoids reporting information that could
- * deanonymize users, though analysis is lacking. */
-#define MIN_IPS_TO_NOTE_COUNTRY 1
-/** Do not report any geoip data at all if we have fewer than this number of
- * IPs to report about. */
-#define MIN_IPS_TO_NOTE_ANYTHING 1
-/** When reporting geoip data about countries, round up to the nearest
- * multiple of this value. */
-#define IP_GRANULARITY 8
-
-/** Helper type: used to sort per-country totals by value. */
-typedef struct c_hist_t {
-  char country[3]; /**< Two-letter country code. */
-  unsigned total; /**< Total IP addresses seen in this country. */
-} c_hist_t;
-
-/** Sorting helper: return -1, 1, or 0 based on comparison of two
- * geoip_ipv4_entry_t.  Sort in descending order of total, and then by country
- * code. */
-static int
-c_hist_compare_(const void **_a, const void **_b)
-{
-  const c_hist_t *a = *_a, *b = *_b;
-  if (a->total > b->total)
-    return -1;
-  else if (a->total < b->total)
-    return 1;
-  else
-    return strcmp(a->country, b->country);
-}
-
-/** When there are incomplete directory requests at the end of a 24-hour
- * period, consider those requests running for longer than this timeout as
- * failed, the others as still running. */
-#define DIRREQ_TIMEOUT (10*60)
-
-/** Entry in a map from either chan->global_identifier for direct requests
- * or a unique circuit identifier for tunneled requests to request time,
- * response size, and completion time of a network status request. Used to
- * measure download times of requests to derive average client
- * bandwidths. */
-typedef struct dirreq_map_entry_t {
-  HT_ENTRY(dirreq_map_entry_t) node;
-  /** Unique identifier for this network status request; this is either the
-   * chan->global_identifier of the dir channel (direct request) or a new
-   * locally unique identifier of a circuit (tunneled request). This ID is
-   * only unique among other direct or tunneled requests, respectively. */
-  uint64_t dirreq_id;
-  unsigned int state:3; /**< State of this directory request. */
-  unsigned int type:1; /**< Is this a direct or a tunneled request? */
-  unsigned int completed:1; /**< Is this request complete? */
-  /** When did we receive the request and started sending the response? */
-  struct timeval request_time;
-  size_t response_size; /**< What is the size of the response in bytes? */
-  struct timeval completion_time; /**< When did the request succeed? */
-} dirreq_map_entry_t;
-
-/** Map of all directory requests asking for v2 or v3 network statuses in
- * the current geoip-stats interval. Values are
- * of type *<b>dirreq_map_entry_t</b>. */
-static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
-     HT_INITIALIZER();
-
-static int
-dirreq_map_ent_eq(const dirreq_map_entry_t *a,
-                  const dirreq_map_entry_t *b)
-{
-  return a->dirreq_id == b->dirreq_id && a->type == b->type;
-}
-
-/* DOCDOC dirreq_map_ent_hash */
-static unsigned
-dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
-{
-  unsigned u = (unsigned) entry->dirreq_id;
-  u += entry->type << 20;
-  return u;
-}
-
-HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq)
-HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
-
-/** Helper: Put <b>entry</b> into map of directory requests using
- * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
- * already an entry for that key, print out a BUG warning and return. */
-static void
-dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
-               uint64_t dirreq_id)
-{
-  dirreq_map_entry_t *old_ent;
-  tor_assert(entry->type == type);
-  tor_assert(entry->dirreq_id == dirreq_id);
-
-  /* XXXX we could switch this to HT_INSERT some time, since it seems that
-   * this bug doesn't happen. But since this function doesn't seem to be
-   * critical-path, it's sane to leave it alone. */
-  old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
-  if (old_ent && old_ent != entry) {
-    log_warn(LD_BUG, "Error when putting directory request into local "
-             "map. There was already an entry for the same identifier.");
-    return;
-  }
-}
-
-/** Helper: Look up and return an entry in the map of directory requests
- * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
- * is no such entry, return NULL. */
-static dirreq_map_entry_t *
-dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
-{
-  dirreq_map_entry_t lookup;
-  lookup.type = type;
-  lookup.dirreq_id = dirreq_id;
-  return HT_FIND(dirreqmap, &dirreq_map, &lookup);
-}
-
-/** Note that an either direct or tunneled (see <b>type</b>) directory
- * request for a v3 network status with unique ID <b>dirreq_id</b> of size
- * <b>response_size</b> has started. */
-void
-geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
-                   dirreq_type_t type)
-{
-  dirreq_map_entry_t *ent;
-  if (!get_options()->DirReqStatistics)
-    return;
-  ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
-  ent->dirreq_id = dirreq_id;
-  tor_gettimeofday(&ent->request_time);
-  ent->response_size = response_size;
-  ent->type = type;
-  dirreq_map_put_(ent, type, dirreq_id);
-}
-
-/** Change the state of the either direct or tunneled (see <b>type</b>)
- * directory request with <b>dirreq_id</b> to <b>new_state</b> and
- * possibly mark it as completed. If no entry can be found for the given
- * key parts (e.g., if this is a directory request that we are not
- * measuring, or one that was started in the previous measurement period),
- * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
-void
-geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
-                          dirreq_state_t new_state)
-{
-  dirreq_map_entry_t *ent;
-  if (!get_options()->DirReqStatistics)
-    return;
-  ent = dirreq_map_get_(type, dirreq_id);
-  if (!ent)
-    return;
-  if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
-    return;
-  if (new_state - 1 != ent->state)
-    return;
-  ent->state = new_state;
-  if ((type == DIRREQ_DIRECT &&
-         new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
-      (type == DIRREQ_TUNNELED &&
-         new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
-    tor_gettimeofday(&ent->completion_time);
-    ent->completed = 1;
-  }
-}
-
-/** Return the bridge-ip-transports string that should be inserted in
- *  our extra-info descriptor. Return NULL if the bridge-ip-transports
- *  line should be empty.  */
-char *
-geoip_get_transport_history(void)
-{
-  unsigned granularity = IP_GRANULARITY;
-  /** String hash table (name of transport) -> (number of users). */
-  strmap_t *transport_counts = strmap_new();
-
-  /** Smartlist that contains copies of the names of the transports
-      that have been used. */
-  smartlist_t *transports_used = smartlist_new();
-
-  /* Special string to signify that no transport was used for this
-     connection. Pluggable transport names can't have symbols in their
-     names, so this string will never collide with a real transport. */
-  static const char* no_transport_str = "<OR>";
-
-  clientmap_entry_t **ent;
-  smartlist_t *string_chunks = smartlist_new();
-  char *the_string = NULL;
-
-  /* If we haven't seen any clients yet, return NULL. */
-  if (HT_EMPTY(&client_history))
-    goto done;
-
-  /** We do the following steps to form the transport history string:
-   *  a) Foreach client that uses a pluggable transport, we increase the
-   *  times that transport was used by one. If the client did not use
-   *  a transport, we increase the number of times someone connected
-   *  without obfuscation.
-   *  b) Foreach transport we observed, we write its transport history
-   *  string and push it to string_chunks. So, for example, if we've
-   *  seen 665 obfs2 clients, we write "obfs2=665".
-   *  c) We concatenate string_chunks to form the final string.
-   */
-
-  log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
-            HT_SIZE(&client_history));
-
-  /* Loop through all clients. */
-  HT_FOREACH(ent, clientmap, &client_history) {
-    uintptr_t val;
-    void *ptr;
-    const char *transport_name = (*ent)->transport_name;
-    if (!transport_name)
-      transport_name = no_transport_str;
-
-    /* Increase the count for this transport name. */
-    ptr = strmap_get(transport_counts, transport_name);
-    val = (uintptr_t)ptr;
-    val++;
-    ptr = (void*)val;
-    strmap_set(transport_counts, transport_name, ptr);
-
-    /* If it's the first time we see this transport, note it. */
-    if (val == 1)
-      smartlist_add_strdup(transports_used, transport_name);
-
-    log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
-              "I've now seen %d clients.",
-              safe_str_client(fmt_addr(&(*ent)->addr)),
-              transport_name ? transport_name : "<no transport>",
-              (int)val);
-  }
-
-  /* Sort the transport names (helps with unit testing). */
-  smartlist_sort_strings(transports_used);
-
-  /* Loop through all seen transports. */
-  SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
-    void *transport_count_ptr = strmap_get(transport_counts, transport_name);
-    uintptr_t transport_count = (uintptr_t) transport_count_ptr;
-
-    log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
-              ((uint64_t)transport_count), transport_name);
-
-    smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
-                           transport_name,
-                           (round_uint64_to_next_multiple_of(
-                                               (uint64_t)transport_count,
-                                               granularity)));
-  } SMARTLIST_FOREACH_END(transport_name);
-
-  the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
-
-  log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
-
- done:
-  strmap_free(transport_counts, NULL);
-  SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
-  smartlist_free(transports_used);
-  SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
-  smartlist_free(string_chunks);
-
-  return the_string;
-}
-
-/** Return a newly allocated comma-separated string containing statistics
- * on network status downloads. The string contains the number of completed
- * requests, timeouts, and still running requests as well as the download
- * times by deciles and quartiles. Return NULL if we have not observed
- * requests for long enough. */
-static char *
-geoip_get_dirreq_history(dirreq_type_t type)
-{
-  char *result = NULL;
-  buf_t *buf = NULL;
-  smartlist_t *dirreq_completed = NULL;
-  uint32_t complete = 0, timeouts = 0, running = 0;
-  dirreq_map_entry_t **ptr, **next;
-  struct timeval now;
-
-  tor_gettimeofday(&now);
-  dirreq_completed = smartlist_new();
-  for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
-    dirreq_map_entry_t *ent = *ptr;
-    if (ent->type != type) {
-      next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
-      continue;
-    } else {
-      if (ent->completed) {
-        smartlist_add(dirreq_completed, ent);
-        complete++;
-        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
-      } else {
-        if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
-          timeouts++;
-        else
-          running++;
-        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
-        tor_free(ent);
-      }
-    }
-  }
-#define DIR_REQ_GRANULARITY 4
-  complete = round_uint32_to_next_multiple_of(complete,
-                                              DIR_REQ_GRANULARITY);
-  timeouts = round_uint32_to_next_multiple_of(timeouts,
-                                              DIR_REQ_GRANULARITY);
-  running = round_uint32_to_next_multiple_of(running,
-                                             DIR_REQ_GRANULARITY);
-  buf = buf_new_with_capacity(1024);
-  buf_add_printf(buf, "complete=%u,timeout=%u,"
-                 "running=%u", complete, timeouts, running);
-
-#define MIN_DIR_REQ_RESPONSES 16
-  if (complete >= MIN_DIR_REQ_RESPONSES) {
-    uint32_t *dltimes;
-    /* We may have rounded 'completed' up.  Here we want to use the
-     * real value. */
-    complete = smartlist_len(dirreq_completed);
-    dltimes = tor_calloc(complete, sizeof(uint32_t));
-    SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
-      uint32_t bytes_per_second;
-      uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time,
-                                               &ent->completion_time);
-      if (time_diff_ == 0)
-        time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible
-                        * by law of nature or something, but a millisecond
-                        * is a bit greater than "instantly" */
-      bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
-      dltimes[ent_sl_idx] = bytes_per_second;
-    } SMARTLIST_FOREACH_END(ent);
-    median_uint32(dltimes, complete); /* sorts as a side effect. */
-    buf_add_printf(buf,
-                           ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
-                           "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
-                           dltimes[0],
-                           dltimes[1*complete/10-1],
-                           dltimes[2*complete/10-1],
-                           dltimes[1*complete/4-1],
-                           dltimes[3*complete/10-1],
-                           dltimes[4*complete/10-1],
-                           dltimes[5*complete/10-1],
-                           dltimes[6*complete/10-1],
-                           dltimes[7*complete/10-1],
-                           dltimes[3*complete/4-1],
-                           dltimes[8*complete/10-1],
-                           dltimes[9*complete/10-1],
-                           dltimes[complete-1]);
-    tor_free(dltimes);
-  }
-
-  result = buf_extract(buf, NULL);
-
-  SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
-                    tor_free(ent));
-  smartlist_free(dirreq_completed);
-  buf_free(buf);
-  return result;
-}
-
-/** Store a newly allocated comma-separated string in
- * *<a>country_str</a> containing entries for all the countries from
- * which we've seen enough clients connect as a bridge, directory
- * server, or entry guard. The entry format is cc=num where num is the
- * number of IPs we've seen connecting from that country, and cc is a
- * lowercased country code. *<a>country_str</a> is set to NULL if
- * we're not ready to export per country data yet.
- *
- * Store a newly allocated comma-separated string in <a>ipver_str</a>
- * containing entries for clients connecting over IPv4 and IPv6. The
- * format is family=num where num is the nubmer of IPs we've seen
- * connecting over that protocol family, and family is 'v4' or 'v6'.
- *
- * Return 0 on success and -1 if we're missing geoip data. */
-int
-geoip_get_client_history(geoip_client_action_t action,
-                         char **country_str, char **ipver_str)
-{
-  unsigned granularity = IP_GRANULARITY;
-  smartlist_t *entries = NULL;
-  int n_countries = geoip_get_n_countries();
-  int i;
-  clientmap_entry_t **cm_ent;
-  unsigned *counts = NULL;
-  unsigned total = 0;
-  unsigned ipv4_count = 0, ipv6_count = 0;
-
-  if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
-    return -1;
-
-  counts = tor_calloc(n_countries, sizeof(unsigned));
-  HT_FOREACH(cm_ent, clientmap, &client_history) {
-    int country;
-    if ((*cm_ent)->action != (int)action)
-      continue;
-    country = geoip_get_country_by_addr(&(*cm_ent)->addr);
-    if (country < 0)
-      country = 0; /** unresolved requests are stored at index 0. */
-    tor_assert(0 <= country && country < n_countries);
-    ++counts[country];
-    ++total;
-    switch (tor_addr_family(&(*cm_ent)->addr)) {
-    case AF_INET:
-      ipv4_count++;
-      break;
-    case AF_INET6:
-      ipv6_count++;
-      break;
-    }
-  }
-  if (ipver_str) {
-    smartlist_t *chunks = smartlist_new();
-    smartlist_add_asprintf(chunks, "v4=%u",
-                           round_to_next_multiple_of(ipv4_count, granularity));
-    smartlist_add_asprintf(chunks, "v6=%u",
-                           round_to_next_multiple_of(ipv6_count, granularity));
-    *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
-    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
-    smartlist_free(chunks);
-  }
-
-  /* Don't record per country data if we haven't seen enough IPs. */
-  if (total < MIN_IPS_TO_NOTE_ANYTHING) {
-    tor_free(counts);
-    if (country_str)
-      *country_str = NULL;
-    return 0;
-  }
-
-  /* Make a list of c_hist_t */
-  entries = smartlist_new();
-  for (i = 0; i < n_countries; ++i) {
-    unsigned c = counts[i];
-    const char *countrycode;
-    c_hist_t *ent;
-    /* Only report a country if it has a minimum number of IPs. */
-    if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
-      c = round_to_next_multiple_of(c, granularity);
-      countrycode = geoip_get_country_name(i);
-      ent = tor_malloc(sizeof(c_hist_t));
-      strlcpy(ent->country, countrycode, sizeof(ent->country));
-      ent->total = c;
-      smartlist_add(entries, ent);
-    }
-  }
-  /* Sort entries. Note that we must do this _AFTER_ rounding, or else
-   * the sort order could leak info. */
-  smartlist_sort(entries, c_hist_compare_);
-
-  if (country_str) {
-    smartlist_t *chunks = smartlist_new();
-    SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
-        smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
-      });
-    *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
-    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
-    smartlist_free(chunks);
-  }
-
-  SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
-  smartlist_free(entries);
-  tor_free(counts);
-
-  return 0;
-}
-
-/** Return a newly allocated string holding the per-country request history
- * for v3 network statuses in a format suitable for an extra-info document,
- * or NULL on failure. */
-char *
-geoip_get_request_history(void)
-{
-  smartlist_t *entries, *strings;
-  char *result;
-  unsigned granularity = IP_GRANULARITY;
-
-  if (!geoip_countries)
-    return NULL;
-
-  entries = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(geoip_countries, geoip_country_t *, c) {
-      uint32_t tot = 0;
-      c_hist_t *ent;
-      if ((size_t)c_sl_idx < n_v3_ns_requests_len)
-        tot = n_v3_ns_requests[c_sl_idx];
-      else
-        tot = 0;
-      if (!tot)
-        continue;
-      ent = tor_malloc_zero(sizeof(c_hist_t));
-      strlcpy(ent->country, c->countrycode, sizeof(ent->country));
-      ent->total = round_to_next_multiple_of(tot, granularity);
-      smartlist_add(entries, ent);
-  } SMARTLIST_FOREACH_END(c);
-  smartlist_sort(entries, c_hist_compare_);
-
-  strings = smartlist_new();
-  SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
-      smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
-  });
-  result = smartlist_join_strings(strings, ",", 0, NULL);
-  SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
-  SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
-  smartlist_free(strings);
-  smartlist_free(entries);
-  return result;
-}
-
-/** Start time of directory request stats or 0 if we're not collecting
- * directory request statistics. */
-static time_t start_of_dirreq_stats_interval;
-
-/** Initialize directory request stats. */
-void
-geoip_dirreq_stats_init(time_t now)
-{
-  start_of_dirreq_stats_interval = now;
-}
-
-/** Reset counters for dirreq stats. */
-void
-geoip_reset_dirreq_stats(time_t now)
-{
-  memset(n_v3_ns_requests, 0,
-         n_v3_ns_requests_len * sizeof(uint32_t));
-  {
-    clientmap_entry_t **ent, **next, *this;
-    for (ent = HT_START(clientmap, &client_history); ent != NULL;
-         ent = next) {
-      if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
-        this = *ent;
-        next = HT_NEXT_RMV(clientmap, &client_history, ent);
-        clientmap_entry_free(this);
-      } else {
-        next = HT_NEXT(clientmap, &client_history, ent);
-      }
-    }
-  }
-  memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
-  {
-    dirreq_map_entry_t **ent, **next, *this;
-    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
-      this = *ent;
-      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
-      tor_free(this);
-    }
-  }
-  start_of_dirreq_stats_interval = now;
-}
-
-/** Stop collecting directory request stats in a way that we can re-start
- * doing so in geoip_dirreq_stats_init(). */
-void
-geoip_dirreq_stats_term(void)
-{
-  geoip_reset_dirreq_stats(0);
-}
-
-/** Return a newly allocated string containing the dirreq statistics
- * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
- * must ensure start_of_dirreq_stats_interval is in the past. */
-char *
-geoip_format_dirreq_stats(time_t now)
-{
-  char t[ISO_TIME_LEN+1];
-  int i;
-  char *v3_ips_string = NULL, *v3_reqs_string = NULL,
-       *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
-  char *result = NULL;
-
-  if (!start_of_dirreq_stats_interval)
-    return NULL; /* Not initialized. */
-
-  tor_assert(now >= start_of_dirreq_stats_interval);
-
-  format_iso_time(t, now);
-  geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
-  v3_reqs_string = geoip_get_request_history();
-
-#define RESPONSE_GRANULARITY 8
-  for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
-    ns_v3_responses[i] = round_uint32_to_next_multiple_of(
-                               ns_v3_responses[i], RESPONSE_GRANULARITY);
-  }
-#undef RESPONSE_GRANULARITY
-
-  v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
-  v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
-
-  /* Put everything together into a single string. */
-  tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
-              "dirreq-v3-ips %s\n"
-              "dirreq-v3-reqs %s\n"
-              "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
-                   "not-found=%u,not-modified=%u,busy=%u\n"
-              "dirreq-v3-direct-dl %s\n"
-              "dirreq-v3-tunneled-dl %s\n",
-              t,
-              (unsigned) (now - start_of_dirreq_stats_interval),
-              v3_ips_string ? v3_ips_string : "",
-              v3_reqs_string ? v3_reqs_string : "",
-              ns_v3_responses[GEOIP_SUCCESS],
-              ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
-              ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
-              ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
-              ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
-              ns_v3_responses[GEOIP_REJECT_BUSY],
-              v3_direct_dl_string ? v3_direct_dl_string : "",
-              v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
-
-  /* Free partial strings. */
-  tor_free(v3_ips_string);
-  tor_free(v3_reqs_string);
-  tor_free(v3_direct_dl_string);
-  tor_free(v3_tunneled_dl_string);
-
-  return result;
-}
-
-/** If 24 hours have passed since the beginning of the current dirreq
- * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
- * (possibly overwriting an existing file) and reset counters.  Return
- * when we would next want to write dirreq stats or 0 if we never want to
- * write. */
-time_t
-geoip_dirreq_stats_write(time_t now)
-{
-  char *str = NULL;
-
-  if (!start_of_dirreq_stats_interval)
-    return 0; /* Not initialized. */
-  if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
-    goto done; /* Not ready to write. */
-
-  /* Discard all items in the client history that are too old. */
-  geoip_remove_old_clients(start_of_dirreq_stats_interval);
-
-  /* Generate history string .*/
-  str = geoip_format_dirreq_stats(now);
-  if (! str)
-    goto done;
-
-  /* Write dirreq-stats string to disk. */
-  if (!check_or_create_data_subdir("stats")) {
-    write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
-    /* Reset measurement interval start. */
-    geoip_reset_dirreq_stats(now);
-  }
-
- done:
-  tor_free(str);
-  return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
-}
-
-/** Start time of bridge stats or 0 if we're not collecting bridge
- * statistics. */
-static time_t start_of_bridge_stats_interval;
-
-/** Initialize bridge stats. */
-void
-geoip_bridge_stats_init(time_t now)
-{
-  start_of_bridge_stats_interval = now;
-}
-
-/** Stop collecting bridge stats in a way that we can re-start doing so in
- * geoip_bridge_stats_init(). */
-void
-geoip_bridge_stats_term(void)
-{
-  client_history_clear();
-  start_of_bridge_stats_interval = 0;
-}
-
-/** Validate a bridge statistics string as it would be written to a
- * current extra-info descriptor. Return 1 if the string is valid and
- * recent enough, or 0 otherwise. */
-static int
-validate_bridge_stats(const char *stats_str, time_t now)
-{
-  char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
-       *eos;
-
-  const char *BRIDGE_STATS_END = "bridge-stats-end ";
-  const char *BRIDGE_IPS = "bridge-ips ";
-  const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
-  const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
-  const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
-  const char *tmp;
-  time_t stats_end_time;
-  int seconds;
-  tor_assert(stats_str);
-
-  /* Parse timestamp and number of seconds from
-    "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
-  tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
-  if (!tmp)
-    return 0;
-  tmp += strlen(BRIDGE_STATS_END);
-
-  if (strlen(tmp) < ISO_TIME_LEN + 6)
-    return 0;
-  strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
-  if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
-    return 0;
-  if (stats_end_time < now - (25*60*60) ||
-      stats_end_time > now + (1*60*60))
-    return 0;
-  seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
-  if (!eos || seconds < 23*60*60)
-    return 0;
-  format_iso_time(stats_start_str, stats_end_time - seconds);
-
-  /* Parse: "bridge-ips CC=N,CC=N,..." */
-  tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
-  if (!tmp) {
-    /* Look if there is an empty "bridge-ips" line */
-    tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
-    if (!tmp)
-      return 0;
-  }
-
-  /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
-  tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
-  if (!tmp) {
-    /* Look if there is an empty "bridge-ip-transports" line */
-    tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
-    if (!tmp)
-      return 0;
-  }
-
-  return 1;
-}
-
-/** Most recent bridge statistics formatted to be written to extra-info
- * descriptors. */
-static char *bridge_stats_extrainfo = NULL;
-
-/** Return a newly allocated string holding our bridge usage stats by country
- * in a format suitable for inclusion in an extrainfo document. Return NULL on
- * failure.  */
-char *
-geoip_format_bridge_stats(time_t now)
-{
-  char *out = NULL;
-  char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
-  long duration = now - start_of_bridge_stats_interval;
-  char written[ISO_TIME_LEN+1];
-
-  if (duration < 0)
-    return NULL;
-  if (!start_of_bridge_stats_interval)
-    return NULL; /* Not initialized. */
-
-  format_iso_time(written, now);
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
-  transport_data = geoip_get_transport_history();
-
-  tor_asprintf(&out,
-               "bridge-stats-end %s (%ld s)\n"
-               "bridge-ips %s\n"
-               "bridge-ip-versions %s\n"
-               "bridge-ip-transports %s\n",
-               written, duration,
-               country_data ? country_data : "",
-               ipver_data ? ipver_data : "",
-               transport_data ? transport_data : "");
-  tor_free(country_data);
-  tor_free(ipver_data);
-  tor_free(transport_data);
-
-  return out;
-}
-
-/** Return a newly allocated string holding our bridge usage stats by country
- * in a format suitable for the answer to a controller request. Return NULL on
- * failure.  */
-static char *
-format_bridge_stats_controller(time_t now)
-{
-  char *out = NULL, *country_data = NULL, *ipver_data = NULL;
-  char started[ISO_TIME_LEN+1];
-  (void) now;
-
-  format_iso_time(started, start_of_bridge_stats_interval);
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
-
-  tor_asprintf(&out,
-               "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
-               started,
-               country_data ? country_data : "",
-               ipver_data ? ipver_data : "");
-  tor_free(country_data);
-  tor_free(ipver_data);
-  return out;
-}
-
-/** Return a newly allocated string holding our bridge usage stats by
- * country in a format suitable for inclusion in our heartbeat
- * message. Return NULL on failure.  */
-char *
-format_client_stats_heartbeat(time_t now)
-{
-  const int n_hours = 6;
-  char *out = NULL;
-  int n_clients = 0;
-  clientmap_entry_t **ent;
-  unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
-
-  if (!start_of_bridge_stats_interval)
-    return NULL; /* Not initialized. */
-
-  /* count unique IPs */
-  HT_FOREACH(ent, clientmap, &client_history) {
-    /* only count directly connecting clients */
-    if ((*ent)->action != GEOIP_CLIENT_CONNECT)
-      continue;
-    if ((*ent)->last_seen_in_minutes < cutoff)
-      continue;
-    n_clients++;
-  }
-
-  tor_asprintf(&out, "Heartbeat: "
-               "In the last %d hours, I have seen %d unique clients.",
-               n_hours,
-               n_clients);
-
-  return out;
-}
-
-/** Write bridge statistics to $DATADIR/stats/bridge-stats and return
- * when we should next try to write statistics. */
-time_t
-geoip_bridge_stats_write(time_t now)
-{
-  char *val = NULL;
-
-  /* Check if 24 hours have passed since starting measurements. */
-  if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
-    return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
-
-  /* Discard all items in the client history that are too old. */
-  geoip_remove_old_clients(start_of_bridge_stats_interval);
-
-  /* Generate formatted string */
-  val = geoip_format_bridge_stats(now);
-  if (val == NULL)
-    goto done;
-
-  /* Update the stored value. */
-  tor_free(bridge_stats_extrainfo);
-  bridge_stats_extrainfo = val;
-  start_of_bridge_stats_interval = now;
-
-  /* Write it to disk. */
-  if (!check_or_create_data_subdir("stats")) {
-    write_to_data_subdir("stats", "bridge-stats",
-                         bridge_stats_extrainfo, "bridge statistics");
-
-    /* Tell the controller, "hey, there are clients!" */
-    {
-      char *controller_str = format_bridge_stats_controller(now);
-      if (controller_str)
-        control_event_clients_seen(controller_str);
-      tor_free(controller_str);
-    }
-  }
-
- done:
-  return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
-}
-
-/** Try to load the most recent bridge statistics from disk, unless we
- * have finished a measurement interval lately, and check whether they
- * are still recent enough. */
-static void
-load_bridge_stats(time_t now)
-{
-  char *fname, *contents;
-  if (bridge_stats_extrainfo)
-    return;
-
-  fname = get_datadir_fname2("stats", "bridge-stats");
-  contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
-  if (contents && validate_bridge_stats(contents, now)) {
-    bridge_stats_extrainfo = contents;
-  } else {
-    tor_free(contents);
-  }
-
-  tor_free(fname);
-}
-
-/** Return most recent bridge statistics for inclusion in extra-info
- * descriptors, or NULL if we don't have recent bridge statistics. */
-const char *
-geoip_get_bridge_stats_extrainfo(time_t now)
-{
-  load_bridge_stats(now);
-  return bridge_stats_extrainfo;
-}
-
-/** Return a new string containing the recent bridge statistics to be returned
- * to controller clients, or NULL if we don't have any bridge statistics. */
-char *
-geoip_get_bridge_stats_controller(time_t now)
-{
-  return format_bridge_stats_controller(now);
-}
-
-/** Start time of entry stats or 0 if we're not collecting entry
- * statistics. */
-static time_t start_of_entry_stats_interval;
-
-/** Initialize entry stats. */
-void
-geoip_entry_stats_init(time_t now)
-{
-  start_of_entry_stats_interval = now;
-}
-
-/** Reset counters for entry stats. */
-void
-geoip_reset_entry_stats(time_t now)
-{
-  client_history_clear();
-  start_of_entry_stats_interval = now;
-}
-
-/** Stop collecting entry stats in a way that we can re-start doing so in
- * geoip_entry_stats_init(). */
-void
-geoip_entry_stats_term(void)
-{
-  geoip_reset_entry_stats(0);
-}
-
-/** Return a newly allocated string containing the entry statistics
- * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
- * must ensure start_of_entry_stats_interval lies in the past. */
-char *
-geoip_format_entry_stats(time_t now)
-{
-  char t[ISO_TIME_LEN+1];
-  char *data = NULL;
-  char *result;
-
-  if (!start_of_entry_stats_interval)
-    return NULL; /* Not initialized. */
-
-  tor_assert(now >= start_of_entry_stats_interval);
-
-  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
-  format_iso_time(t, now);
-  tor_asprintf(&result,
-               "entry-stats-end %s (%u s)\n"
-               "entry-ips %s\n",
-               t, (unsigned) (now - start_of_entry_stats_interval),
-               data ? data : "");
-  tor_free(data);
-  return result;
-}
-
-/** If 24 hours have passed since the beginning of the current entry stats
- * period, write entry stats to $DATADIR/stats/entry-stats (possibly
- * overwriting an existing file) and reset counters.  Return when we would
- * next want to write entry stats or 0 if we never want to write. */
-time_t
-geoip_entry_stats_write(time_t now)
-{
-  char *str = NULL;
-
-  if (!start_of_entry_stats_interval)
-    return 0; /* Not initialized. */
-  if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
-    goto done; /* Not ready to write. */
-
-  /* Discard all items in the client history that are too old. */
-  geoip_remove_old_clients(start_of_entry_stats_interval);
-
-  /* Generate history string .*/
-  str = geoip_format_entry_stats(now);
-
-  /* Write entry-stats string to disk. */
-  if (!check_or_create_data_subdir("stats")) {
-    write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
-
-    /* Reset measurement interval start. */
-    geoip_reset_entry_stats(now);
-  }
-
- done:
-  tor_free(str);
-  return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
-}
-
-/** Helper used to implement GETINFO ip-to-country/... controller command. */
-int
-getinfo_helper_geoip(control_connection_t *control_conn,
-                     const char *question, char **answer,
-                     const char **errmsg)
-{
-  (void)control_conn;
-  if (!strcmpstart(question, "ip-to-country/")) {
-    int c;
-    sa_family_t family;
-    tor_addr_t addr;
-    question += strlen("ip-to-country/");
-
-    if (!strcmp(question, "ipv4-available") ||
-        !strcmp(question, "ipv6-available")) {
-      family = !strcmp(question, "ipv4-available") ? AF_INET : AF_INET6;
-      const int available = geoip_is_loaded(family);
-      tor_asprintf(answer, "%d", !! available);
-      return 0;
-    }
-
-    family = tor_addr_parse(&addr, question);
-    if (family != AF_INET && family != AF_INET6) {
-      *errmsg = "Invalid address family";
-      return -1;
-    }
-    if (!geoip_is_loaded(family)) {
-      *errmsg = "GeoIP data not loaded";
-      return -1;
-    }
-    if (family == AF_INET)
-      c = geoip_get_country_by_ipv4(tor_addr_to_ipv4h(&addr));
-    else                      /* AF_INET6 */
-      c = geoip_get_country_by_ipv6(tor_addr_to_in6(&addr));
-    *answer = tor_strdup(geoip_get_country_name(c));
-  }
-  return 0;
-}
-
 /** Release all storage held by the GeoIP databases and country list. */
 STATIC void
 clear_geoip_db(void)
@@ -1892,28 +500,7 @@ clear_geoip_db(void)
 void
 geoip_free_all(void)
 {
-  {
-    clientmap_entry_t **ent, **next, *this;
-    for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
-      this = *ent;
-      next = HT_NEXT_RMV(clientmap, &client_history, ent);
-      clientmap_entry_free(this);
-    }
-    HT_CLEAR(clientmap, &client_history);
-  }
-  {
-    dirreq_map_entry_t **ent, **next, *this;
-    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
-      this = *ent;
-      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
-      tor_free(this);
-    }
-    HT_CLEAR(dirreqmap, &dirreq_map);
-  }
-
   clear_geoip_db();
-  tor_free(bridge_stats_extrainfo);
-  tor_free(n_v3_ns_requests);
 
   memset(geoip_digest, 0, sizeof(geoip_digest));
   memset(geoip6_digest, 0, sizeof(geoip6_digest));
diff --git a/src/feature/stats/geoip.h b/src/feature/stats/geoip.h
index a2175c01e..4adfc10fb 100644
--- a/src/feature/stats/geoip.h
+++ b/src/feature/stats/geoip.h
@@ -13,96 +13,22 @@
 #define TOR_GEOIP_H
 
 #include "lib/testsupport/testsupport.h"
-#include "core/or/dos.h"
-
-/** Indicates an action that we might be noting geoip statistics on.
- * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
- * the others, we're not.
- */
-typedef enum {
-  /** We've noticed a connection as a bridge relay or entry guard. */
-  GEOIP_CLIENT_CONNECT = 0,
-  /** We've served a networkstatus consensus as a directory server. */
-  GEOIP_CLIENT_NETWORKSTATUS = 1,
-} geoip_client_action_t;
-/** Indicates either a positive reply or a reason for rejectng a network
- * status request that will be included in geoip statistics. */
-typedef enum {
-  /** Request is answered successfully. */
-  GEOIP_SUCCESS = 0,
-  /** V3 network status is not signed by a sufficient number of requested
-   * authorities. */
-  GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
-  /** Requested network status object is unavailable. */
-  GEOIP_REJECT_UNAVAILABLE = 2,
-  /** Requested network status not found. */
-  GEOIP_REJECT_NOT_FOUND = 3,
-  /** Network status has not been modified since If-Modified-Since time. */
-  GEOIP_REJECT_NOT_MODIFIED = 4,
-  /** Directory is busy. */
-  GEOIP_REJECT_BUSY = 5,
-} geoip_ns_response_t;
-#define GEOIP_NS_RESPONSE_NUM 6
-
-/** Directory requests that we are measuring can be either direct or
- * tunneled. */
-typedef enum {
-  DIRREQ_DIRECT = 0,
-  DIRREQ_TUNNELED = 1,
-} dirreq_type_t;
-
-/** Possible states for either direct or tunneled directory requests that
- * are relevant for determining network status download times. */
-typedef enum {
-  /** Found that the client requests a network status; applies to both
-   * direct and tunneled requests; initial state of a request that we are
-   * measuring. */
-  DIRREQ_IS_FOR_NETWORK_STATUS = 0,
-  /** Finished writing a network status to the directory connection;
-   * applies to both direct and tunneled requests; completes a direct
-   * request. */
-  DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
-  /** END cell sent to circuit that initiated a tunneled request. */
-  DIRREQ_END_CELL_SENT = 2,
-  /** Flushed last cell from queue of the circuit that initiated a
-    * tunneled request to the outbuf of the OR connection. */
-  DIRREQ_CIRC_QUEUE_FLUSHED = 3,
-  /** Flushed last byte from buffer of the channel belonging to the
-    * circuit that initiated a tunneled request; completes a tunneled
-    * request. */
-  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
-} dirreq_state_t;
 
 #ifdef GEOIP_PRIVATE
 STATIC int geoip_parse_entry(const char *line, sa_family_t family);
-STATIC int geoip_get_country_by_ipv4(uint32_t ipaddr);
-STATIC int geoip_get_country_by_ipv6(const struct in6_addr *addr);
 STATIC void clear_geoip_db(void);
 #endif /* defined(GEOIP_PRIVATE) */
 
-/** Entry in a map from IP address to the last time we've seen an incoming
- * connection from that IP address. Used by bridges only to track which
- * countries have them blocked, or the DoS mitigation subsystem if enabled. */
-typedef struct clientmap_entry_t {
-  HT_ENTRY(clientmap_entry_t) node;
-  tor_addr_t addr;
-  /* Name of pluggable transport used by this client. NULL if no
-     pluggable transport was used. */
-  char *transport_name;
+int geoip_get_country_by_ipv4(uint32_t ipaddr);
+int geoip_get_country_by_ipv6(const struct in6_addr *addr);
 
-  /** Time when we last saw this IP address, in MINUTES since the epoch.
-   *
-   * (This will run out of space around 4011 CE.  If Tor is still in use around
-   * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
-  unsigned int last_seen_in_minutes:30;
-  unsigned int action:2;
+/** A per-country GeoIP record. */
+typedef struct geoip_country_t {
+  char countrycode[3];
+} geoip_country_t;
 
-  /* This object is used to keep some statistics per client address for the
-   * DoS mitigation subsystem. */
-  dos_client_stats_t dos_stats;
-} clientmap_entry_t;
+const smartlist_t *geoip_get_countries(void);
 
-int should_record_bridge_info(const or_options_t *options);
 int geoip_load_file(sa_family_t family, const char *filename);
 MOCK_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
 MOCK_DECL(int, geoip_get_n_countries, (void));
@@ -111,47 +37,6 @@ MOCK_DECL(int, geoip_is_loaded, (sa_family_t family));
 const char *geoip_db_digest(sa_family_t family);
 MOCK_DECL(country_t, geoip_get_country, (const char *countrycode));
 
-void geoip_note_client_seen(geoip_client_action_t action,
-                            const tor_addr_t *addr, const char *transport_name,
-                            time_t now);
-void geoip_remove_old_clients(time_t cutoff);
-clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr,
-                                       const char *transport_name,
-                                       geoip_client_action_t action);
-size_t geoip_client_cache_total_allocation(void);
-size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
-
-void geoip_note_ns_response(geoip_ns_response_t response);
-char *geoip_get_transport_history(void);
-int geoip_get_client_history(geoip_client_action_t action,
-                             char **country_str, char **ipver_str);
-char *geoip_get_request_history(void);
-int getinfo_helper_geoip(control_connection_t *control_conn,
-                         const char *question, char **answer,
-                         const char **errmsg);
 void geoip_free_all(void);
 
-void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
-                        dirreq_type_t type);
-void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
-                               dirreq_state_t new_state);
-
-void geoip_dirreq_stats_init(time_t now);
-void geoip_reset_dirreq_stats(time_t now);
-char *geoip_format_dirreq_stats(time_t now);
-time_t geoip_dirreq_stats_write(time_t now);
-void geoip_dirreq_stats_term(void);
-void geoip_entry_stats_init(time_t now);
-time_t geoip_entry_stats_write(time_t now);
-void geoip_entry_stats_term(void);
-void geoip_reset_entry_stats(time_t now);
-char *geoip_format_entry_stats(time_t now);
-void geoip_bridge_stats_init(time_t now);
-char *geoip_format_bridge_stats(time_t now);
-time_t geoip_bridge_stats_write(time_t now);
-void geoip_bridge_stats_term(void);
-const char *geoip_get_bridge_stats_extrainfo(time_t);
-char *geoip_get_bridge_stats_controller(time_t);
-char *format_client_stats_heartbeat(time_t now);
-
 #endif /* !defined(TOR_GEOIP_H) */
diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c
new file mode 100644
index 000000000..4af2ff0ec
--- /dev/null
+++ b/src/feature/stats/geoip_stats.c
@@ -0,0 +1,1422 @@
+/* Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip.c
+ * \brief Functions related to maintaining an IP-to-country database;
+ * to summarizing client connections by country to entry guards, bridges,
+ * and directory servers; and for statistics on answering network status
+ * requests.
+ *
+ * There are two main kinds of functions in this module: geoip functions,
+ * which map groups of IPv4 and IPv6 addresses to country codes, and
+ * statistical functions, which collect statistics about different kinds of
+ * per-country usage.
+ *
+ * The geoip lookup tables are implemented as sorted lists of disjoint address
+ * ranges, each mapping to a singleton geoip_country_t.  These country objects
+ * are also indexed by their names in a hashtable.
+ *
+ * The tables are populated from disk at startup by the geoip_load_file()
+ * function.  For more information on the file format they read, see that
+ * function.  See the scripts and the README file in src/config for more
+ * information about how those files are generated.
+ *
+ * Tor uses GeoIP information in order to implement user requests (such as
+ * ExcludeNodes {cc}), and to keep track of how much usage relays are getting
+ * for each country.
+ */
+
+#include "core/or/or.h"
+
+#include "ht.h"
+#include "lib/container/buffers.h"
+#include "app/config/config.h"
+#include "feature/control/control.h"
+#include "feature/client/dnsserv.h"
+#include "core/or/dos.h"
+#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "lib/container/order.h"
+#include "lib/time/tvdiff.h"
+
+/** Number of entries in n_v3_ns_requests */
+size_t n_v3_ns_requests_len = 0;
+/** Array, indexed by country index, of number of v3 networkstatus requests
+ * received from that country */
+static uint32_t *n_v3_ns_requests;
+
+/* Total size in bytes of the geoip client history cache. Used by the OOM
+ * handler. */
+static size_t geoip_client_history_cache_size;
+
+/* Increment the geoip client history cache size counter with the given bytes.
+ * This prevents an overflow and set it to its maximum in that case. */
+static inline void
+geoip_increment_client_history_cache_size(size_t bytes)
+{
+  /* This is shockingly high, lets log it so it can be reported. */
+  IF_BUG_ONCE(geoip_client_history_cache_size > (SIZE_MAX - bytes)) {
+    geoip_client_history_cache_size = SIZE_MAX;
+    return;
+  }
+  geoip_client_history_cache_size += bytes;
+}
+
+/* Decrement the geoip client history cache size counter with the given bytes.
+ * This prevents an underflow and set it to 0 in that case. */
+static inline void
+geoip_decrement_client_history_cache_size(size_t bytes)
+{
+  /* Going below 0 means that we either allocated an entry without
+   * incrementing the counter or we have different sizes when allocating and
+   * freeing. It shouldn't happened so log it. */
+  IF_BUG_ONCE(geoip_client_history_cache_size < bytes) {
+    geoip_client_history_cache_size = 0;
+    return;
+  }
+  geoip_client_history_cache_size -= bytes;
+}
+
+/** Add 1 to the count of v3 ns requests received from <b>country</b>. */
+static void
+increment_v3_ns_request(country_t country)
+{
+  if (country < 0)
+    return;
+
+  if ((size_t)country >= n_v3_ns_requests_len) {
+    /* We need to reallocate the array. */
+    size_t new_len;
+    if (n_v3_ns_requests_len == 0)
+      new_len = 256;
+    else
+      new_len = n_v3_ns_requests_len * 2;
+    if (new_len <= (size_t)country)
+      new_len = ((size_t)country)+1;
+    n_v3_ns_requests = tor_reallocarray(n_v3_ns_requests, new_len,
+                                        sizeof(uint32_t));
+    memset(n_v3_ns_requests + n_v3_ns_requests_len, 0,
+           sizeof(uint32_t)*(new_len - n_v3_ns_requests_len));
+    n_v3_ns_requests_len = new_len;
+  }
+
+  n_v3_ns_requests[country] += 1;
+}
+
+/** Return 1 if we should collect geoip stats on bridge users, and
+ * include them in our extrainfo descriptor. Else return 0. */
+int
+should_record_bridge_info(const or_options_t *options)
+{
+  return options->BridgeRelay && options->BridgeRecordUsageByCountry;
+}
+
+/** Largest allowable value for last_seen_in_minutes.  (It's a 30-bit field,
+ * so it can hold up to (1u<<30)-1, or 0x3fffffffu.
+ */
+#define MAX_LAST_SEEN_IN_MINUTES 0X3FFFFFFFu
+
+/** Map from client IP address to last time seen. */
+static HT_HEAD(clientmap, clientmap_entry_t) client_history =
+     HT_INITIALIZER();
+
+/** Hashtable helper: compute a hash of a clientmap_entry_t. */
+static inline unsigned
+clientmap_entry_hash(const clientmap_entry_t *a)
+{
+  unsigned h = (unsigned) tor_addr_hash(&a->addr);
+
+  if (a->transport_name)
+    h += (unsigned) siphash24g(a->transport_name, strlen(a->transport_name));
+
+  return h;
+}
+/** Hashtable helper: compare two clientmap_entry_t values for equality. */
+static inline int
+clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
+{
+  if (strcmp_opt(a->transport_name, b->transport_name))
+    return 0;
+
+  return !tor_addr_compare(&a->addr, &b->addr, CMP_EXACT) &&
+         a->action == b->action;
+}
+
+HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
+             clientmap_entries_eq)
+HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
+             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+
+#define clientmap_entry_free(ent) \
+  FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
+
+/** Return the size of a client map entry. */
+static inline size_t
+clientmap_entry_size(const clientmap_entry_t *ent)
+{
+  tor_assert(ent);
+  return (sizeof(clientmap_entry_t) +
+          (ent->transport_name ? strlen(ent->transport_name) : 0));
+}
+
+/** Free all storage held by <b>ent</b>. */
+static void
+clientmap_entry_free_(clientmap_entry_t *ent)
+{
+  if (!ent)
+    return;
+
+  /* This entry is about to be freed so pass it to the DoS subsystem to see if
+   * any actions can be taken about it. */
+  dos_geoip_entry_about_to_free(ent);
+  geoip_decrement_client_history_cache_size(clientmap_entry_size(ent));
+
+  tor_free(ent->transport_name);
+  tor_free(ent);
+}
+
+/* Return a newly allocated clientmap entry with the given action and address
+ * that are mandatory. The transport_name can be optional. This can't fail. */
+static clientmap_entry_t *
+clientmap_entry_new(geoip_client_action_t action, const tor_addr_t *addr,
+                    const char *transport_name)
+{
+  clientmap_entry_t *entry;
+
+  tor_assert(action == GEOIP_CLIENT_CONNECT ||
+             action == GEOIP_CLIENT_NETWORKSTATUS);
+  tor_assert(addr);
+
+  entry = tor_malloc_zero(sizeof(clientmap_entry_t));
+  entry->action = action;
+  tor_addr_copy(&entry->addr, addr);
+  if (transport_name) {
+    entry->transport_name = tor_strdup(transport_name);
+  }
+
+  /* Allocated and initialized, note down its size for the OOM handler. */
+  geoip_increment_client_history_cache_size(clientmap_entry_size(entry));
+
+  return entry;
+}
+
+/** Clear history of connecting clients used by entry and bridge stats. */
+static void
+client_history_clear(void)
+{
+  clientmap_entry_t **ent, **next, *this;
+  for (ent = HT_START(clientmap, &client_history); ent != NULL;
+       ent = next) {
+    if ((*ent)->action == GEOIP_CLIENT_CONNECT) {
+      this = *ent;
+      next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      clientmap_entry_free(this);
+    } else {
+      next = HT_NEXT(clientmap, &client_history, ent);
+    }
+  }
+}
+
+/** Note that we've seen a client connect from the IP <b>addr</b>
+ * at time <b>now</b>. Ignored by all but bridges and directories if
+ * configured accordingly. */
+void
+geoip_note_client_seen(geoip_client_action_t action,
+                       const tor_addr_t *addr,
+                       const char *transport_name,
+                       time_t now)
+{
+  const or_options_t *options = get_options();
+  clientmap_entry_t *ent;
+
+  if (action == GEOIP_CLIENT_CONNECT) {
+    /* Only remember statistics if the DoS mitigation subsystem is enabled. If
+     * not, only if as entry guard or as bridge. */
+    if (!dos_enabled()) {
+      if (!options->EntryStatistics && !should_record_bridge_info(options)) {
+        return;
+      }
+    }
+  } else {
+    /* Only gather directory-request statistics if configured, and
+     * forcibly disable them on bridge authorities. */
+    if (!options->DirReqStatistics || options->BridgeAuthoritativeDir)
+      return;
+  }
+
+  log_debug(LD_GENERAL, "Seen client from '%s' with transport '%s'.",
+            safe_str_client(fmt_addr((addr))),
+            transport_name ? transport_name : "<no transport>");
+
+  ent = geoip_lookup_client(addr, transport_name, action);
+  if (! ent) {
+    ent = clientmap_entry_new(action, addr, transport_name);
+    HT_INSERT(clientmap, &client_history, ent);
+  }
+  if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
+    ent->last_seen_in_minutes = (unsigned)(now/60);
+  else
+    ent->last_seen_in_minutes = 0;
+
+  if (action == GEOIP_CLIENT_NETWORKSTATUS) {
+    int country_idx = geoip_get_country_by_addr(addr);
+    if (country_idx < 0)
+      country_idx = 0; /** unresolved requests are stored at index 0. */
+    increment_v3_ns_request(country_idx);
+  }
+}
+
+/** HT_FOREACH helper: remove a clientmap_entry_t from the hashtable if it's
+ * older than a certain time. */
+static int
+remove_old_client_helper_(struct clientmap_entry_t *ent, void *_cutoff)
+{
+  time_t cutoff = *(time_t*)_cutoff / 60;
+  if (ent->last_seen_in_minutes < cutoff) {
+    clientmap_entry_free(ent);
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+/** Forget about all clients that haven't connected since <b>cutoff</b>. */
+void
+geoip_remove_old_clients(time_t cutoff)
+{
+  clientmap_HT_FOREACH_FN(&client_history,
+                          remove_old_client_helper_,
+                          &cutoff);
+}
+
+/* Return a client entry object matching the given address, transport name and
+ * geoip action from the clientmap. NULL if not found. The transport_name can
+ * be NULL. */
+clientmap_entry_t *
+geoip_lookup_client(const tor_addr_t *addr, const char *transport_name,
+                    geoip_client_action_t action)
+{
+  clientmap_entry_t lookup;
+
+  tor_assert(addr);
+
+  /* We always look for a client connection with no transport. */
+  tor_addr_copy(&lookup.addr, addr);
+  lookup.action = action;
+  lookup.transport_name = (char *) transport_name;
+
+  return HT_FIND(clientmap, &client_history, &lookup);
+}
+
+/* Cleanup client entries older than the cutoff. Used for the OOM. Return the
+ * number of bytes freed. If 0 is returned, nothing was freed. */
+static size_t
+oom_clean_client_entries(time_t cutoff)
+{
+  size_t bytes = 0;
+  clientmap_entry_t **ent, **ent_next;
+
+  for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
+    clientmap_entry_t *entry = *ent;
+    if (entry->last_seen_in_minutes < (cutoff / 60)) {
+      ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      bytes += clientmap_entry_size(entry);
+      clientmap_entry_free(entry);
+    } else {
+      ent_next = HT_NEXT(clientmap, &client_history, ent);
+    }
+  }
+  return bytes;
+}
+
+/* Below this minimum lifetime, the OOM won't cleanup any entries. */
+#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
+/* The OOM moves the cutoff by that much every run. */
+#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
+
+/* Cleanup the geoip client history cache called from the OOM handler. Return
+ * the amount of bytes removed. This can return a value below or above
+ * min_remove_bytes but will stop as oon as the min_remove_bytes has been
+ * reached. */
+size_t
+geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+  time_t k;
+  size_t bytes_removed = 0;
+
+  /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+  tor_assert(min_remove_bytes != 0);
+
+  /* Set k to the initial cutoff of an entry. We then going to move it by step
+   * to try to remove as much as we can. */
+  k = WRITE_STATS_INTERVAL;
+
+  do {
+    time_t cutoff;
+
+    /* If k has reached the minimum lifetime, we have to stop else we might
+     * remove every single entries which would be pretty bad for the DoS
+     * mitigation subsystem if by just filling the geoip cache, it was enough
+     * to trigger the OOM and clean every single entries. */
+    if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
+      break;
+    }
+
+    cutoff = now - k;
+    bytes_removed += oom_clean_client_entries(cutoff);
+    k -= GEOIP_CLIENT_CACHE_OOM_STEP;
+  } while (bytes_removed < min_remove_bytes);
+
+  return bytes_removed;
+}
+
+/* Return the total size in bytes of the client history cache. */
+size_t
+geoip_client_cache_total_allocation(void)
+{
+  return geoip_client_history_cache_size;
+}
+
+/** How many responses are we giving to clients requesting v3 network
+ * statuses? */
+static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
+
+/** Note that we've rejected a client's request for a v3 network status
+ * for reason <b>reason</b> at time <b>now</b>. */
+void
+geoip_note_ns_response(geoip_ns_response_t response)
+{
+  static int arrays_initialized = 0;
+  if (!get_options()->DirReqStatistics)
+    return;
+  if (!arrays_initialized) {
+    memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
+    arrays_initialized = 1;
+  }
+  tor_assert(response < GEOIP_NS_RESPONSE_NUM);
+  ns_v3_responses[response]++;
+}
+
+/** Do not mention any country from which fewer than this number of IPs have
+ * connected.  This conceivably avoids reporting information that could
+ * deanonymize users, though analysis is lacking. */
+#define MIN_IPS_TO_NOTE_COUNTRY 1
+/** Do not report any geoip data at all if we have fewer than this number of
+ * IPs to report about. */
+#define MIN_IPS_TO_NOTE_ANYTHING 1
+/** When reporting geoip data about countries, round up to the nearest
+ * multiple of this value. */
+#define IP_GRANULARITY 8
+
+/** Helper type: used to sort per-country totals by value. */
+typedef struct c_hist_t {
+  char country[3]; /**< Two-letter country code. */
+  unsigned total; /**< Total IP addresses seen in this country. */
+} c_hist_t;
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * geoip_ipv4_entry_t.  Sort in descending order of total, and then by country
+ * code. */
+static int
+c_hist_compare_(const void **_a, const void **_b)
+{
+  const c_hist_t *a = *_a, *b = *_b;
+  if (a->total > b->total)
+    return -1;
+  else if (a->total < b->total)
+    return 1;
+  else
+    return strcmp(a->country, b->country);
+}
+
+/** When there are incomplete directory requests at the end of a 24-hour
+ * period, consider those requests running for longer than this timeout as
+ * failed, the others as still running. */
+#define DIRREQ_TIMEOUT (10*60)
+
+/** Entry in a map from either chan->global_identifier for direct requests
+ * or a unique circuit identifier for tunneled requests to request time,
+ * response size, and completion time of a network status request. Used to
+ * measure download times of requests to derive average client
+ * bandwidths. */
+typedef struct dirreq_map_entry_t {
+  HT_ENTRY(dirreq_map_entry_t) node;
+  /** Unique identifier for this network status request; this is either the
+   * chan->global_identifier of the dir channel (direct request) or a new
+   * locally unique identifier of a circuit (tunneled request). This ID is
+   * only unique among other direct or tunneled requests, respectively. */
+  uint64_t dirreq_id;
+  unsigned int state:3; /**< State of this directory request. */
+  unsigned int type:1; /**< Is this a direct or a tunneled request? */
+  unsigned int completed:1; /**< Is this request complete? */
+  /** When did we receive the request and started sending the response? */
+  struct timeval request_time;
+  size_t response_size; /**< What is the size of the response in bytes? */
+  struct timeval completion_time; /**< When did the request succeed? */
+} dirreq_map_entry_t;
+
+/** Map of all directory requests asking for v2 or v3 network statuses in
+ * the current geoip-stats interval. Values are
+ * of type *<b>dirreq_map_entry_t</b>. */
+static HT_HEAD(dirreqmap, dirreq_map_entry_t) dirreq_map =
+     HT_INITIALIZER();
+
+static int
+dirreq_map_ent_eq(const dirreq_map_entry_t *a,
+                  const dirreq_map_entry_t *b)
+{
+  return a->dirreq_id == b->dirreq_id && a->type == b->type;
+}
+
+/* DOCDOC dirreq_map_ent_hash */
+static unsigned
+dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
+{
+  unsigned u = (unsigned) entry->dirreq_id;
+  u += entry->type << 20;
+  return u;
+}
+
+HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
+             dirreq_map_ent_eq)
+HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
+             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Helper: Put <b>entry</b> into map of directory requests using
+ * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
+ * already an entry for that key, print out a BUG warning and return. */
+static void
+dirreq_map_put_(dirreq_map_entry_t *entry, dirreq_type_t type,
+               uint64_t dirreq_id)
+{
+  dirreq_map_entry_t *old_ent;
+  tor_assert(entry->type == type);
+  tor_assert(entry->dirreq_id == dirreq_id);
+
+  /* XXXX we could switch this to HT_INSERT some time, since it seems that
+   * this bug doesn't happen. But since this function doesn't seem to be
+   * critical-path, it's sane to leave it alone. */
+  old_ent = HT_REPLACE(dirreqmap, &dirreq_map, entry);
+  if (old_ent && old_ent != entry) {
+    log_warn(LD_BUG, "Error when putting directory request into local "
+             "map. There was already an entry for the same identifier.");
+    return;
+  }
+}
+
+/** Helper: Look up and return an entry in the map of directory requests
+ * using <b>type</b> and <b>dirreq_id</b> as key parts. If there
+ * is no such entry, return NULL. */
+static dirreq_map_entry_t *
+dirreq_map_get_(dirreq_type_t type, uint64_t dirreq_id)
+{
+  dirreq_map_entry_t lookup;
+  lookup.type = type;
+  lookup.dirreq_id = dirreq_id;
+  return HT_FIND(dirreqmap, &dirreq_map, &lookup);
+}
+
+/** Note that an either direct or tunneled (see <b>type</b>) directory
+ * request for a v3 network status with unique ID <b>dirreq_id</b> of size
+ * <b>response_size</b> has started. */
+void
+geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
+                   dirreq_type_t type)
+{
+  dirreq_map_entry_t *ent;
+  if (!get_options()->DirReqStatistics)
+    return;
+  ent = tor_malloc_zero(sizeof(dirreq_map_entry_t));
+  ent->dirreq_id = dirreq_id;
+  tor_gettimeofday(&ent->request_time);
+  ent->response_size = response_size;
+  ent->type = type;
+  dirreq_map_put_(ent, type, dirreq_id);
+}
+
+/** Change the state of the either direct or tunneled (see <b>type</b>)
+ * directory request with <b>dirreq_id</b> to <b>new_state</b> and
+ * possibly mark it as completed. If no entry can be found for the given
+ * key parts (e.g., if this is a directory request that we are not
+ * measuring, or one that was started in the previous measurement period),
+ * or if the state cannot be advanced to <b>new_state</b>, do nothing. */
+void
+geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
+                          dirreq_state_t new_state)
+{
+  dirreq_map_entry_t *ent;
+  if (!get_options()->DirReqStatistics)
+    return;
+  ent = dirreq_map_get_(type, dirreq_id);
+  if (!ent)
+    return;
+  if (new_state == DIRREQ_IS_FOR_NETWORK_STATUS)
+    return;
+  if (new_state - 1 != ent->state)
+    return;
+  ent->state = new_state;
+  if ((type == DIRREQ_DIRECT &&
+         new_state == DIRREQ_FLUSHING_DIR_CONN_FINISHED) ||
+      (type == DIRREQ_TUNNELED &&
+         new_state == DIRREQ_CHANNEL_BUFFER_FLUSHED)) {
+    tor_gettimeofday(&ent->completion_time);
+    ent->completed = 1;
+  }
+}
+
+/** Return the bridge-ip-transports string that should be inserted in
+ *  our extra-info descriptor. Return NULL if the bridge-ip-transports
+ *  line should be empty.  */
+char *
+geoip_get_transport_history(void)
+{
+  unsigned granularity = IP_GRANULARITY;
+  /** String hash table (name of transport) -> (number of users). */
+  strmap_t *transport_counts = strmap_new();
+
+  /** Smartlist that contains copies of the names of the transports
+      that have been used. */
+  smartlist_t *transports_used = smartlist_new();
+
+  /* Special string to signify that no transport was used for this
+     connection. Pluggable transport names can't have symbols in their
+     names, so this string will never collide with a real transport. */
+  static const char* no_transport_str = "<OR>";
+
+  clientmap_entry_t **ent;
+  smartlist_t *string_chunks = smartlist_new();
+  char *the_string = NULL;
+
+  /* If we haven't seen any clients yet, return NULL. */
+  if (HT_EMPTY(&client_history))
+    goto done;
+
+  /** We do the following steps to form the transport history string:
+   *  a) Foreach client that uses a pluggable transport, we increase the
+   *  times that transport was used by one. If the client did not use
+   *  a transport, we increase the number of times someone connected
+   *  without obfuscation.
+   *  b) Foreach transport we observed, we write its transport history
+   *  string and push it to string_chunks. So, for example, if we've
+   *  seen 665 obfs2 clients, we write "obfs2=665".
+   *  c) We concatenate string_chunks to form the final string.
+   */
+
+  log_debug(LD_GENERAL,"Starting iteration for transport history. %d clients.",
+            HT_SIZE(&client_history));
+
+  /* Loop through all clients. */
+  HT_FOREACH(ent, clientmap, &client_history) {
+    uintptr_t val;
+    void *ptr;
+    const char *transport_name = (*ent)->transport_name;
+    if (!transport_name)
+      transport_name = no_transport_str;
+
+    /* Increase the count for this transport name. */
+    ptr = strmap_get(transport_counts, transport_name);
+    val = (uintptr_t)ptr;
+    val++;
+    ptr = (void*)val;
+    strmap_set(transport_counts, transport_name, ptr);
+
+    /* If it's the first time we see this transport, note it. */
+    if (val == 1)
+      smartlist_add_strdup(transports_used, transport_name);
+
+    log_debug(LD_GENERAL, "Client from '%s' with transport '%s'. "
+              "I've now seen %d clients.",
+              safe_str_client(fmt_addr(&(*ent)->addr)),
+              transport_name ? transport_name : "<no transport>",
+              (int)val);
+  }
+
+  /* Sort the transport names (helps with unit testing). */
+  smartlist_sort_strings(transports_used);
+
+  /* Loop through all seen transports. */
+  SMARTLIST_FOREACH_BEGIN(transports_used, const char *, transport_name) {
+    void *transport_count_ptr = strmap_get(transport_counts, transport_name);
+    uintptr_t transport_count = (uintptr_t) transport_count_ptr;
+
+    log_debug(LD_GENERAL, "We got %"PRIu64" clients with transport '%s'.",
+              ((uint64_t)transport_count), transport_name);
+
+    smartlist_add_asprintf(string_chunks, "%s=%"PRIu64,
+                           transport_name,
+                           (round_uint64_to_next_multiple_of(
+                                               (uint64_t)transport_count,
+                                               granularity)));
+  } SMARTLIST_FOREACH_END(transport_name);
+
+  the_string = smartlist_join_strings(string_chunks, ",", 0, NULL);
+
+  log_debug(LD_GENERAL, "Final bridge-ip-transports string: '%s'", the_string);
+
+ done:
+  strmap_free(transport_counts, NULL);
+  SMARTLIST_FOREACH(transports_used, char *, s, tor_free(s));
+  smartlist_free(transports_used);
+  SMARTLIST_FOREACH(string_chunks, char *, s, tor_free(s));
+  smartlist_free(string_chunks);
+
+  return the_string;
+}
+
+/** Return a newly allocated comma-separated string containing statistics
+ * on network status downloads. The string contains the number of completed
+ * requests, timeouts, and still running requests as well as the download
+ * times by deciles and quartiles. Return NULL if we have not observed
+ * requests for long enough. */
+static char *
+geoip_get_dirreq_history(dirreq_type_t type)
+{
+  char *result = NULL;
+  buf_t *buf = NULL;
+  smartlist_t *dirreq_completed = NULL;
+  uint32_t complete = 0, timeouts = 0, running = 0;
+  dirreq_map_entry_t **ptr, **next;
+  struct timeval now;
+
+  tor_gettimeofday(&now);
+  dirreq_completed = smartlist_new();
+  for (ptr = HT_START(dirreqmap, &dirreq_map); ptr; ptr = next) {
+    dirreq_map_entry_t *ent = *ptr;
+    if (ent->type != type) {
+      next = HT_NEXT(dirreqmap, &dirreq_map, ptr);
+      continue;
+    } else {
+      if (ent->completed) {
+        smartlist_add(dirreq_completed, ent);
+        complete++;
+        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
+      } else {
+        if (tv_mdiff(&ent->request_time, &now) / 1000 > DIRREQ_TIMEOUT)
+          timeouts++;
+        else
+          running++;
+        next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ptr);
+        tor_free(ent);
+      }
+    }
+  }
+#define DIR_REQ_GRANULARITY 4
+  complete = round_uint32_to_next_multiple_of(complete,
+                                              DIR_REQ_GRANULARITY);
+  timeouts = round_uint32_to_next_multiple_of(timeouts,
+                                              DIR_REQ_GRANULARITY);
+  running = round_uint32_to_next_multiple_of(running,
+                                             DIR_REQ_GRANULARITY);
+  buf = buf_new_with_capacity(1024);
+  buf_add_printf(buf, "complete=%u,timeout=%u,"
+                 "running=%u", complete, timeouts, running);
+
+#define MIN_DIR_REQ_RESPONSES 16
+  if (complete >= MIN_DIR_REQ_RESPONSES) {
+    uint32_t *dltimes;
+    /* We may have rounded 'completed' up.  Here we want to use the
+     * real value. */
+    complete = smartlist_len(dirreq_completed);
+    dltimes = tor_calloc(complete, sizeof(uint32_t));
+    SMARTLIST_FOREACH_BEGIN(dirreq_completed, dirreq_map_entry_t *, ent) {
+      uint32_t bytes_per_second;
+      uint32_t time_diff_ = (uint32_t) tv_mdiff(&ent->request_time,
+                                               &ent->completion_time);
+      if (time_diff_ == 0)
+        time_diff_ = 1; /* Avoid DIV/0; "instant" answers are impossible
+                        * by law of nature or something, but a millisecond
+                        * is a bit greater than "instantly" */
+      bytes_per_second = (uint32_t)(1000 * ent->response_size / time_diff_);
+      dltimes[ent_sl_idx] = bytes_per_second;
+    } SMARTLIST_FOREACH_END(ent);
+    median_uint32(dltimes, complete); /* sorts as a side effect. */
+    buf_add_printf(buf,
+                           ",min=%u,d1=%u,d2=%u,q1=%u,d3=%u,d4=%u,md=%u,"
+                           "d6=%u,d7=%u,q3=%u,d8=%u,d9=%u,max=%u",
+                           dltimes[0],
+                           dltimes[1*complete/10-1],
+                           dltimes[2*complete/10-1],
+                           dltimes[1*complete/4-1],
+                           dltimes[3*complete/10-1],
+                           dltimes[4*complete/10-1],
+                           dltimes[5*complete/10-1],
+                           dltimes[6*complete/10-1],
+                           dltimes[7*complete/10-1],
+                           dltimes[3*complete/4-1],
+                           dltimes[8*complete/10-1],
+                           dltimes[9*complete/10-1],
+                           dltimes[complete-1]);
+    tor_free(dltimes);
+  }
+
+  result = buf_extract(buf, NULL);
+
+  SMARTLIST_FOREACH(dirreq_completed, dirreq_map_entry_t *, ent,
+                    tor_free(ent));
+  smartlist_free(dirreq_completed);
+  buf_free(buf);
+  return result;
+}
+
+/** Store a newly allocated comma-separated string in
+ * *<a>country_str</a> containing entries for all the countries from
+ * which we've seen enough clients connect as a bridge, directory
+ * server, or entry guard. The entry format is cc=num where num is the
+ * number of IPs we've seen connecting from that country, and cc is a
+ * lowercased country code. *<a>country_str</a> is set to NULL if
+ * we're not ready to export per country data yet.
+ *
+ * Store a newly allocated comma-separated string in <a>ipver_str</a>
+ * containing entries for clients connecting over IPv4 and IPv6. The
+ * format is family=num where num is the nubmer of IPs we've seen
+ * connecting over that protocol family, and family is 'v4' or 'v6'.
+ *
+ * Return 0 on success and -1 if we're missing geoip data. */
+int
+geoip_get_client_history(geoip_client_action_t action,
+                         char **country_str, char **ipver_str)
+{
+  unsigned granularity = IP_GRANULARITY;
+  smartlist_t *entries = NULL;
+  int n_countries = geoip_get_n_countries();
+  int i;
+  clientmap_entry_t **cm_ent;
+  unsigned *counts = NULL;
+  unsigned total = 0;
+  unsigned ipv4_count = 0, ipv6_count = 0;
+
+  if (!geoip_is_loaded(AF_INET) && !geoip_is_loaded(AF_INET6))
+    return -1;
+
+  counts = tor_calloc(n_countries, sizeof(unsigned));
+  HT_FOREACH(cm_ent, clientmap, &client_history) {
+    int country;
+    if ((*cm_ent)->action != (int)action)
+      continue;
+    country = geoip_get_country_by_addr(&(*cm_ent)->addr);
+    if (country < 0)
+      country = 0; /** unresolved requests are stored at index 0. */
+    tor_assert(0 <= country && country < n_countries);
+    ++counts[country];
+    ++total;
+    switch (tor_addr_family(&(*cm_ent)->addr)) {
+    case AF_INET:
+      ipv4_count++;
+      break;
+    case AF_INET6:
+      ipv6_count++;
+      break;
+    }
+  }
+  if (ipver_str) {
+    smartlist_t *chunks = smartlist_new();
+    smartlist_add_asprintf(chunks, "v4=%u",
+                           round_to_next_multiple_of(ipv4_count, granularity));
+    smartlist_add_asprintf(chunks, "v6=%u",
+                           round_to_next_multiple_of(ipv6_count, granularity));
+    *ipver_str = smartlist_join_strings(chunks, ",", 0, NULL);
+    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
+    smartlist_free(chunks);
+  }
+
+  /* Don't record per country data if we haven't seen enough IPs. */
+  if (total < MIN_IPS_TO_NOTE_ANYTHING) {
+    tor_free(counts);
+    if (country_str)
+      *country_str = NULL;
+    return 0;
+  }
+
+  /* Make a list of c_hist_t */
+  entries = smartlist_new();
+  for (i = 0; i < n_countries; ++i) {
+    unsigned c = counts[i];
+    const char *countrycode;
+    c_hist_t *ent;
+    /* Only report a country if it has a minimum number of IPs. */
+    if (c >= MIN_IPS_TO_NOTE_COUNTRY) {
+      c = round_to_next_multiple_of(c, granularity);
+      countrycode = geoip_get_country_name(i);
+      ent = tor_malloc(sizeof(c_hist_t));
+      strlcpy(ent->country, countrycode, sizeof(ent->country));
+      ent->total = c;
+      smartlist_add(entries, ent);
+    }
+  }
+  /* Sort entries. Note that we must do this _AFTER_ rounding, or else
+   * the sort order could leak info. */
+  smartlist_sort(entries, c_hist_compare_);
+
+  if (country_str) {
+    smartlist_t *chunks = smartlist_new();
+    SMARTLIST_FOREACH(entries, c_hist_t *, ch, {
+        smartlist_add_asprintf(chunks, "%s=%u", ch->country, ch->total);
+      });
+    *country_str = smartlist_join_strings(chunks, ",", 0, NULL);
+    SMARTLIST_FOREACH(chunks, char *, c, tor_free(c));
+    smartlist_free(chunks);
+  }
+
+  SMARTLIST_FOREACH(entries, c_hist_t *, c, tor_free(c));
+  smartlist_free(entries);
+  tor_free(counts);
+
+  return 0;
+}
+
+/** Return a newly allocated string holding the per-country request history
+ * for v3 network statuses in a format suitable for an extra-info document,
+ * or NULL on failure. */
+char *
+geoip_get_request_history(void)
+{
+  smartlist_t *entries, *strings;
+  char *result;
+  unsigned granularity = IP_GRANULARITY;
+
+  entries = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(geoip_get_countries(), const geoip_country_t *, c) {
+      uint32_t tot = 0;
+      c_hist_t *ent;
+      if ((size_t)c_sl_idx < n_v3_ns_requests_len)
+        tot = n_v3_ns_requests[c_sl_idx];
+      else
+        tot = 0;
+      if (!tot)
+        continue;
+      ent = tor_malloc_zero(sizeof(c_hist_t));
+      strlcpy(ent->country, c->countrycode, sizeof(ent->country));
+      ent->total = round_to_next_multiple_of(tot, granularity);
+      smartlist_add(entries, ent);
+  } SMARTLIST_FOREACH_END(c);
+  smartlist_sort(entries, c_hist_compare_);
+
+  strings = smartlist_new();
+  SMARTLIST_FOREACH(entries, c_hist_t *, ent, {
+      smartlist_add_asprintf(strings, "%s=%u", ent->country, ent->total);
+  });
+  result = smartlist_join_strings(strings, ",", 0, NULL);
+  SMARTLIST_FOREACH(strings, char *, cp, tor_free(cp));
+  SMARTLIST_FOREACH(entries, c_hist_t *, ent, tor_free(ent));
+  smartlist_free(strings);
+  smartlist_free(entries);
+  return result;
+}
+
+/** Start time of directory request stats or 0 if we're not collecting
+ * directory request statistics. */
+static time_t start_of_dirreq_stats_interval;
+
+/** Initialize directory request stats. */
+void
+geoip_dirreq_stats_init(time_t now)
+{
+  start_of_dirreq_stats_interval = now;
+}
+
+/** Reset counters for dirreq stats. */
+void
+geoip_reset_dirreq_stats(time_t now)
+{
+  memset(n_v3_ns_requests, 0,
+         n_v3_ns_requests_len * sizeof(uint32_t));
+  {
+    clientmap_entry_t **ent, **next, *this;
+    for (ent = HT_START(clientmap, &client_history); ent != NULL;
+         ent = next) {
+      if ((*ent)->action == GEOIP_CLIENT_NETWORKSTATUS) {
+        this = *ent;
+        next = HT_NEXT_RMV(clientmap, &client_history, ent);
+        clientmap_entry_free(this);
+      } else {
+        next = HT_NEXT(clientmap, &client_history, ent);
+      }
+    }
+  }
+  memset(ns_v3_responses, 0, sizeof(ns_v3_responses));
+  {
+    dirreq_map_entry_t **ent, **next, *this;
+    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
+      tor_free(this);
+    }
+  }
+  start_of_dirreq_stats_interval = now;
+}
+
+/** Stop collecting directory request stats in a way that we can re-start
+ * doing so in geoip_dirreq_stats_init(). */
+void
+geoip_dirreq_stats_term(void)
+{
+  geoip_reset_dirreq_stats(0);
+}
+
+/** Return a newly allocated string containing the dirreq statistics
+ * until <b>now</b>, or NULL if we're not collecting dirreq stats. Caller
+ * must ensure start_of_dirreq_stats_interval is in the past. */
+char *
+geoip_format_dirreq_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  int i;
+  char *v3_ips_string = NULL, *v3_reqs_string = NULL,
+       *v3_direct_dl_string = NULL, *v3_tunneled_dl_string = NULL;
+  char *result = NULL;
+
+  if (!start_of_dirreq_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_dirreq_stats_interval);
+
+  format_iso_time(t, now);
+  geoip_get_client_history(GEOIP_CLIENT_NETWORKSTATUS, &v3_ips_string, NULL);
+  v3_reqs_string = geoip_get_request_history();
+
+#define RESPONSE_GRANULARITY 8
+  for (i = 0; i < GEOIP_NS_RESPONSE_NUM; i++) {
+    ns_v3_responses[i] = round_uint32_to_next_multiple_of(
+                               ns_v3_responses[i], RESPONSE_GRANULARITY);
+  }
+#undef RESPONSE_GRANULARITY
+
+  v3_direct_dl_string = geoip_get_dirreq_history(DIRREQ_DIRECT);
+  v3_tunneled_dl_string = geoip_get_dirreq_history(DIRREQ_TUNNELED);
+
+  /* Put everything together into a single string. */
+  tor_asprintf(&result, "dirreq-stats-end %s (%d s)\n"
+              "dirreq-v3-ips %s\n"
+              "dirreq-v3-reqs %s\n"
+              "dirreq-v3-resp ok=%u,not-enough-sigs=%u,unavailable=%u,"
+                   "not-found=%u,not-modified=%u,busy=%u\n"
+              "dirreq-v3-direct-dl %s\n"
+              "dirreq-v3-tunneled-dl %s\n",
+              t,
+              (unsigned) (now - start_of_dirreq_stats_interval),
+              v3_ips_string ? v3_ips_string : "",
+              v3_reqs_string ? v3_reqs_string : "",
+              ns_v3_responses[GEOIP_SUCCESS],
+              ns_v3_responses[GEOIP_REJECT_NOT_ENOUGH_SIGS],
+              ns_v3_responses[GEOIP_REJECT_UNAVAILABLE],
+              ns_v3_responses[GEOIP_REJECT_NOT_FOUND],
+              ns_v3_responses[GEOIP_REJECT_NOT_MODIFIED],
+              ns_v3_responses[GEOIP_REJECT_BUSY],
+              v3_direct_dl_string ? v3_direct_dl_string : "",
+              v3_tunneled_dl_string ? v3_tunneled_dl_string : "");
+
+  /* Free partial strings. */
+  tor_free(v3_ips_string);
+  tor_free(v3_reqs_string);
+  tor_free(v3_direct_dl_string);
+  tor_free(v3_tunneled_dl_string);
+
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current dirreq
+ * stats period, write dirreq stats to $DATADIR/stats/dirreq-stats
+ * (possibly overwriting an existing file) and reset counters.  Return
+ * when we would next want to write dirreq stats or 0 if we never want to
+ * write. */
+time_t
+geoip_dirreq_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_dirreq_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write. */
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_dirreq_stats_interval);
+
+  /* Generate history string .*/
+  str = geoip_format_dirreq_stats(now);
+  if (! str)
+    goto done;
+
+  /* Write dirreq-stats string to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "dirreq-stats", str, "dirreq statistics");
+    /* Reset measurement interval start. */
+    geoip_reset_dirreq_stats(now);
+  }
+
+ done:
+  tor_free(str);
+  return start_of_dirreq_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Start time of bridge stats or 0 if we're not collecting bridge
+ * statistics. */
+static time_t start_of_bridge_stats_interval;
+
+/** Initialize bridge stats. */
+void
+geoip_bridge_stats_init(time_t now)
+{
+  start_of_bridge_stats_interval = now;
+}
+
+/** Stop collecting bridge stats in a way that we can re-start doing so in
+ * geoip_bridge_stats_init(). */
+void
+geoip_bridge_stats_term(void)
+{
+  client_history_clear();
+  start_of_bridge_stats_interval = 0;
+}
+
+/** Validate a bridge statistics string as it would be written to a
+ * current extra-info descriptor. Return 1 if the string is valid and
+ * recent enough, or 0 otherwise. */
+static int
+validate_bridge_stats(const char *stats_str, time_t now)
+{
+  char stats_end_str[ISO_TIME_LEN+1], stats_start_str[ISO_TIME_LEN+1],
+       *eos;
+
+  const char *BRIDGE_STATS_END = "bridge-stats-end ";
+  const char *BRIDGE_IPS = "bridge-ips ";
+  const char *BRIDGE_IPS_EMPTY_LINE = "bridge-ips\n";
+  const char *BRIDGE_TRANSPORTS = "bridge-ip-transports ";
+  const char *BRIDGE_TRANSPORTS_EMPTY_LINE = "bridge-ip-transports\n";
+  const char *tmp;
+  time_t stats_end_time;
+  int seconds;
+  tor_assert(stats_str);
+
+  /* Parse timestamp and number of seconds from
+    "bridge-stats-end YYYY-MM-DD HH:MM:SS (N s)" */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_STATS_END);
+  if (!tmp)
+    return 0;
+  tmp += strlen(BRIDGE_STATS_END);
+
+  if (strlen(tmp) < ISO_TIME_LEN + 6)
+    return 0;
+  strlcpy(stats_end_str, tmp, sizeof(stats_end_str));
+  if (parse_iso_time(stats_end_str, &stats_end_time) < 0)
+    return 0;
+  if (stats_end_time < now - (25*60*60) ||
+      stats_end_time > now + (1*60*60))
+    return 0;
+  seconds = (int)strtol(tmp + ISO_TIME_LEN + 2, &eos, 10);
+  if (!eos || seconds < 23*60*60)
+    return 0;
+  format_iso_time(stats_start_str, stats_end_time - seconds);
+
+  /* Parse: "bridge-ips CC=N,CC=N,..." */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS);
+  if (!tmp) {
+    /* Look if there is an empty "bridge-ips" line */
+    tmp = find_str_at_start_of_line(stats_str, BRIDGE_IPS_EMPTY_LINE);
+    if (!tmp)
+      return 0;
+  }
+
+  /* Parse: "bridge-ip-transports PT=N,PT=N,..." */
+  tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS);
+  if (!tmp) {
+    /* Look if there is an empty "bridge-ip-transports" line */
+    tmp = find_str_at_start_of_line(stats_str, BRIDGE_TRANSPORTS_EMPTY_LINE);
+    if (!tmp)
+      return 0;
+  }
+
+  return 1;
+}
+
+/** Most recent bridge statistics formatted to be written to extra-info
+ * descriptors. */
+static char *bridge_stats_extrainfo = NULL;
+
+/** Return a newly allocated string holding our bridge usage stats by country
+ * in a format suitable for inclusion in an extrainfo document. Return NULL on
+ * failure.  */
+char *
+geoip_format_bridge_stats(time_t now)
+{
+  char *out = NULL;
+  char *country_data = NULL, *ipver_data = NULL, *transport_data = NULL;
+  long duration = now - start_of_bridge_stats_interval;
+  char written[ISO_TIME_LEN+1];
+
+  if (duration < 0)
+    return NULL;
+  if (!start_of_bridge_stats_interval)
+    return NULL; /* Not initialized. */
+
+  format_iso_time(written, now);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
+  transport_data = geoip_get_transport_history();
+
+  tor_asprintf(&out,
+               "bridge-stats-end %s (%ld s)\n"
+               "bridge-ips %s\n"
+               "bridge-ip-versions %s\n"
+               "bridge-ip-transports %s\n",
+               written, duration,
+               country_data ? country_data : "",
+               ipver_data ? ipver_data : "",
+               transport_data ? transport_data : "");
+  tor_free(country_data);
+  tor_free(ipver_data);
+  tor_free(transport_data);
+
+  return out;
+}
+
+/** Return a newly allocated string holding our bridge usage stats by country
+ * in a format suitable for the answer to a controller request. Return NULL on
+ * failure.  */
+static char *
+format_bridge_stats_controller(time_t now)
+{
+  char *out = NULL, *country_data = NULL, *ipver_data = NULL;
+  char started[ISO_TIME_LEN+1];
+  (void) now;
+
+  format_iso_time(started, start_of_bridge_stats_interval);
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &country_data, &ipver_data);
+
+  tor_asprintf(&out,
+               "TimeStarted=\"%s\" CountrySummary=%s IPVersions=%s",
+               started,
+               country_data ? country_data : "",
+               ipver_data ? ipver_data : "");
+  tor_free(country_data);
+  tor_free(ipver_data);
+  return out;
+}
+
+/** Return a newly allocated string holding our bridge usage stats by
+ * country in a format suitable for inclusion in our heartbeat
+ * message. Return NULL on failure.  */
+char *
+format_client_stats_heartbeat(time_t now)
+{
+  const int n_hours = 6;
+  char *out = NULL;
+  int n_clients = 0;
+  clientmap_entry_t **ent;
+  unsigned cutoff = (unsigned)( (now-n_hours*3600)/60 );
+
+  if (!start_of_bridge_stats_interval)
+    return NULL; /* Not initialized. */
+
+  /* count unique IPs */
+  HT_FOREACH(ent, clientmap, &client_history) {
+    /* only count directly connecting clients */
+    if ((*ent)->action != GEOIP_CLIENT_CONNECT)
+      continue;
+    if ((*ent)->last_seen_in_minutes < cutoff)
+      continue;
+    n_clients++;
+  }
+
+  tor_asprintf(&out, "Heartbeat: "
+               "In the last %d hours, I have seen %d unique clients.",
+               n_hours,
+               n_clients);
+
+  return out;
+}
+
+/** Write bridge statistics to $DATADIR/stats/bridge-stats and return
+ * when we should next try to write statistics. */
+time_t
+geoip_bridge_stats_write(time_t now)
+{
+  char *val = NULL;
+
+  /* Check if 24 hours have passed since starting measurements. */
+  if (now < start_of_bridge_stats_interval + WRITE_STATS_INTERVAL)
+    return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_bridge_stats_interval);
+
+  /* Generate formatted string */
+  val = geoip_format_bridge_stats(now);
+  if (val == NULL)
+    goto done;
+
+  /* Update the stored value. */
+  tor_free(bridge_stats_extrainfo);
+  bridge_stats_extrainfo = val;
+  start_of_bridge_stats_interval = now;
+
+  /* Write it to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "bridge-stats",
+                         bridge_stats_extrainfo, "bridge statistics");
+
+    /* Tell the controller, "hey, there are clients!" */
+    {
+      char *controller_str = format_bridge_stats_controller(now);
+      if (controller_str)
+        control_event_clients_seen(controller_str);
+      tor_free(controller_str);
+    }
+  }
+
+ done:
+  return start_of_bridge_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Try to load the most recent bridge statistics from disk, unless we
+ * have finished a measurement interval lately, and check whether they
+ * are still recent enough. */
+static void
+load_bridge_stats(time_t now)
+{
+  char *fname, *contents;
+  if (bridge_stats_extrainfo)
+    return;
+
+  fname = get_datadir_fname2("stats", "bridge-stats");
+  contents = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+  if (contents && validate_bridge_stats(contents, now)) {
+    bridge_stats_extrainfo = contents;
+  } else {
+    tor_free(contents);
+  }
+
+  tor_free(fname);
+}
+
+/** Return most recent bridge statistics for inclusion in extra-info
+ * descriptors, or NULL if we don't have recent bridge statistics. */
+const char *
+geoip_get_bridge_stats_extrainfo(time_t now)
+{
+  load_bridge_stats(now);
+  return bridge_stats_extrainfo;
+}
+
+/** Return a new string containing the recent bridge statistics to be returned
+ * to controller clients, or NULL if we don't have any bridge statistics. */
+char *
+geoip_get_bridge_stats_controller(time_t now)
+{
+  return format_bridge_stats_controller(now);
+}
+
+/** Start time of entry stats or 0 if we're not collecting entry
+ * statistics. */
+static time_t start_of_entry_stats_interval;
+
+/** Initialize entry stats. */
+void
+geoip_entry_stats_init(time_t now)
+{
+  start_of_entry_stats_interval = now;
+}
+
+/** Reset counters for entry stats. */
+void
+geoip_reset_entry_stats(time_t now)
+{
+  client_history_clear();
+  start_of_entry_stats_interval = now;
+}
+
+/** Stop collecting entry stats in a way that we can re-start doing so in
+ * geoip_entry_stats_init(). */
+void
+geoip_entry_stats_term(void)
+{
+  geoip_reset_entry_stats(0);
+}
+
+/** Return a newly allocated string containing the entry statistics
+ * until <b>now</b>, or NULL if we're not collecting entry stats. Caller
+ * must ensure start_of_entry_stats_interval lies in the past. */
+char *
+geoip_format_entry_stats(time_t now)
+{
+  char t[ISO_TIME_LEN+1];
+  char *data = NULL;
+  char *result;
+
+  if (!start_of_entry_stats_interval)
+    return NULL; /* Not initialized. */
+
+  tor_assert(now >= start_of_entry_stats_interval);
+
+  geoip_get_client_history(GEOIP_CLIENT_CONNECT, &data, NULL);
+  format_iso_time(t, now);
+  tor_asprintf(&result,
+               "entry-stats-end %s (%u s)\n"
+               "entry-ips %s\n",
+               t, (unsigned) (now - start_of_entry_stats_interval),
+               data ? data : "");
+  tor_free(data);
+  return result;
+}
+
+/** If 24 hours have passed since the beginning of the current entry stats
+ * period, write entry stats to $DATADIR/stats/entry-stats (possibly
+ * overwriting an existing file) and reset counters.  Return when we would
+ * next want to write entry stats or 0 if we never want to write. */
+time_t
+geoip_entry_stats_write(time_t now)
+{
+  char *str = NULL;
+
+  if (!start_of_entry_stats_interval)
+    return 0; /* Not initialized. */
+  if (start_of_entry_stats_interval + WRITE_STATS_INTERVAL > now)
+    goto done; /* Not ready to write. */
+
+  /* Discard all items in the client history that are too old. */
+  geoip_remove_old_clients(start_of_entry_stats_interval);
+
+  /* Generate history string .*/
+  str = geoip_format_entry_stats(now);
+
+  /* Write entry-stats string to disk. */
+  if (!check_or_create_data_subdir("stats")) {
+    write_to_data_subdir("stats", "entry-stats", str, "entry statistics");
+
+    /* Reset measurement interval start. */
+    geoip_reset_entry_stats(now);
+  }
+
+ done:
+  tor_free(str);
+  return start_of_entry_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Release all storage held in this file. */
+void
+geoip_stats_free_all(void)
+{
+  {
+    clientmap_entry_t **ent, **next, *this;
+    for (ent = HT_START(clientmap, &client_history); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(clientmap, &client_history, ent);
+      clientmap_entry_free(this);
+    }
+    HT_CLEAR(clientmap, &client_history);
+  }
+  {
+    dirreq_map_entry_t **ent, **next, *this;
+    for (ent = HT_START(dirreqmap, &dirreq_map); ent != NULL; ent = next) {
+      this = *ent;
+      next = HT_NEXT_RMV(dirreqmap, &dirreq_map, ent);
+      tor_free(this);
+    }
+    HT_CLEAR(dirreqmap, &dirreq_map);
+  }
+
+  tor_free(bridge_stats_extrainfo);
+  tor_free(n_v3_ns_requests);
+}
diff --git a/src/feature/stats/geoip_stats.h b/src/feature/stats/geoip_stats.h
new file mode 100644
index 000000000..97011d62c
--- /dev/null
+++ b/src/feature/stats/geoip_stats.h
@@ -0,0 +1,139 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file geoip_stats.h
+ * \brief Header file for geoip_stats.c.
+ **/
+
+#ifndef TOR_GEOIP_STATS_H
+#define TOR_GEOIP_STATS_H
+
+#include "core/or/dos.h"
+
+/** Indicates an action that we might be noting geoip statistics on.
+ * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
+ * the others, we're not.
+ */
+typedef enum {
+  /** We've noticed a connection as a bridge relay or entry guard. */
+  GEOIP_CLIENT_CONNECT = 0,
+  /** We've served a networkstatus consensus as a directory server. */
+  GEOIP_CLIENT_NETWORKSTATUS = 1,
+} geoip_client_action_t;
+/** Indicates either a positive reply or a reason for rejectng a network
+ * status request that will be included in geoip statistics. */
+typedef enum {
+  /** Request is answered successfully. */
+  GEOIP_SUCCESS = 0,
+  /** V3 network status is not signed by a sufficient number of requested
+   * authorities. */
+  GEOIP_REJECT_NOT_ENOUGH_SIGS = 1,
+  /** Requested network status object is unavailable. */
+  GEOIP_REJECT_UNAVAILABLE = 2,
+  /** Requested network status not found. */
+  GEOIP_REJECT_NOT_FOUND = 3,
+  /** Network status has not been modified since If-Modified-Since time. */
+  GEOIP_REJECT_NOT_MODIFIED = 4,
+  /** Directory is busy. */
+  GEOIP_REJECT_BUSY = 5,
+} geoip_ns_response_t;
+#define GEOIP_NS_RESPONSE_NUM 6
+
+/** Directory requests that we are measuring can be either direct or
+ * tunneled. */
+typedef enum {
+  DIRREQ_DIRECT = 0,
+  DIRREQ_TUNNELED = 1,
+} dirreq_type_t;
+
+/** Possible states for either direct or tunneled directory requests that
+ * are relevant for determining network status download times. */
+typedef enum {
+  /** Found that the client requests a network status; applies to both
+   * direct and tunneled requests; initial state of a request that we are
+   * measuring. */
+  DIRREQ_IS_FOR_NETWORK_STATUS = 0,
+  /** Finished writing a network status to the directory connection;
+   * applies to both direct and tunneled requests; completes a direct
+   * request. */
+  DIRREQ_FLUSHING_DIR_CONN_FINISHED = 1,
+  /** END cell sent to circuit that initiated a tunneled request. */
+  DIRREQ_END_CELL_SENT = 2,
+  /** Flushed last cell from queue of the circuit that initiated a
+    * tunneled request to the outbuf of the OR connection. */
+  DIRREQ_CIRC_QUEUE_FLUSHED = 3,
+  /** Flushed last byte from buffer of the channel belonging to the
+    * circuit that initiated a tunneled request; completes a tunneled
+    * request. */
+  DIRREQ_CHANNEL_BUFFER_FLUSHED = 4
+} dirreq_state_t;
+
+/** Entry in a map from IP address to the last time we've seen an incoming
+ * connection from that IP address. Used by bridges only to track which
+ * countries have them blocked, or the DoS mitigation subsystem if enabled. */
+typedef struct clientmap_entry_t {
+  HT_ENTRY(clientmap_entry_t) node;
+  tor_addr_t addr;
+  /* Name of pluggable transport used by this client. NULL if no
+     pluggable transport was used. */
+  char *transport_name;
+
+  /** Time when we last saw this IP address, in MINUTES since the epoch.
+   *
+   * (This will run out of space around 4011 CE.  If Tor is still in use around
+   * 4000 CE, please remember to add more bits to last_seen_in_minutes.) */
+  unsigned int last_seen_in_minutes:30;
+  unsigned int action:2;
+
+  /* This object is used to keep some statistics per client address for the
+   * DoS mitigation subsystem. */
+  dos_client_stats_t dos_stats;
+} clientmap_entry_t;
+
+int should_record_bridge_info(const or_options_t *options);
+
+void geoip_note_client_seen(geoip_client_action_t action,
+                            const tor_addr_t *addr, const char *transport_name,
+                            time_t now);
+void geoip_remove_old_clients(time_t cutoff);
+clientmap_entry_t *geoip_lookup_client(const tor_addr_t *addr,
+                                       const char *transport_name,
+                                       geoip_client_action_t action);
+size_t geoip_client_cache_total_allocation(void);
+size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
+
+void geoip_note_ns_response(geoip_ns_response_t response);
+char *geoip_get_transport_history(void);
+int geoip_get_client_history(geoip_client_action_t action,
+                             char **country_str, char **ipver_str);
+char *geoip_get_request_history(void);
+void geoip_stats_free_all(void);
+
+void geoip_start_dirreq(uint64_t dirreq_id, size_t response_size,
+                        dirreq_type_t type);
+void geoip_change_dirreq_state(uint64_t dirreq_id, dirreq_type_t type,
+                               dirreq_state_t new_state);
+
+void geoip_dirreq_stats_init(time_t now);
+void geoip_reset_dirreq_stats(time_t now);
+char *geoip_format_dirreq_stats(time_t now);
+time_t geoip_dirreq_stats_write(time_t now);
+void geoip_dirreq_stats_term(void);
+void geoip_entry_stats_init(time_t now);
+time_t geoip_entry_stats_write(time_t now);
+void geoip_entry_stats_term(void);
+void geoip_reset_entry_stats(time_t now);
+char *geoip_format_entry_stats(time_t now);
+void geoip_bridge_stats_init(time_t now);
+char *geoip_format_bridge_stats(time_t now);
+time_t geoip_bridge_stats_write(time_t now);
+void geoip_bridge_stats_term(void);
+const char *geoip_get_bridge_stats_extrainfo(time_t);
+char *geoip_get_bridge_stats_controller(time_t);
+char *format_client_stats_heartbeat(time_t now);
+
+#endif /* !defined(TOR_GEOIP_STATS_H) */
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index f05b0fc79..7f7a00abd 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -33,6 +33,7 @@
 #include "feature/nodelist/networkstatus.h"
 #include "core/proto/proto_http.h"
 #include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/dirvote.h"
 #include "test/log_test_helpers.h"
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index b411e7b38..40a4c6ba2 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -9,7 +9,7 @@
 #include "core/or/dos.h"
 #include "core/or/circuitlist.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "core/or/channel.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
@@ -500,4 +500,3 @@ struct testcase_t dos_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c
index 9df8ea798..8f13939db 100644
--- a/src/test/test_geoip.c
+++ b/src/test/test_geoip.c
@@ -11,6 +11,7 @@
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "feature/stats/geoip.h"
+#include "feature/stats/geoip_stats.h"
 #include "test/test.h"
 
   /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
Prev by Author: [tor-commits] [tor/maint-0.3.4] Merge branch 'maint-0.3.3' into maint-0.3.4
Next by Author: [tor-commits] [tor/master] Avoid mentioning ticket number in comments
Previous by thread: [tor-commits] [tor/master] Move the n_v3_ns_requests field out of geoip_country_t
Next by thread: [tor-commits] [tor/master] Remove excess dependencies from geoip.c
Index(es):
- Author
- Thread