[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [tor/release-0.3.2] geoip: Hook the client history cache into the OOM handler
commit 51839f47650463f59bd2cc84da05d5bc535d699d
Author: David Goulet <dgoulet@xxxxxxxxxxxxxx>
Date: Fri Feb 2 10:15:28 2018 -0500
geoip: Hook the client history cache into the OOM handler
If the cache is using 20% of our maximum allowed memory, clean 10% of it. Same
behavior as the HS descriptor cache.
Closes #25122
Signed-off-by: David Goulet <dgoulet@xxxxxxxxxxxxxx>
---
changes/ticket25122 | 4 +++
src/or/geoip.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++
src/or/geoip.h | 2 ++
src/or/relay.c | 16 ++++++++--
src/test/test.c | 18 +++++++++++
5 files changed, 128 insertions(+), 3 deletions(-)
diff --git a/changes/ticket25122 b/changes/ticket25122
new file mode 100644
index 000000000..2921811b2
--- /dev/null
+++ b/changes/ticket25122
@@ -0,0 +1,4 @@
+ o Minor feature (geoip cache):
+ - Make our OOM handler aware of the geoip client history cache so it
+ doesn't fill up the memory which is especially important for IPv6 and
+ our DoS mitigation subsystem. Closes ticket 25122.
diff --git a/src/or/geoip.c b/src/or/geoip.c
index 00c055bbe..c5e8cdab9 100644
--- a/src/or/geoip.c
+++ b/src/or/geoip.c
@@ -72,6 +72,10 @@ static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
static char geoip_digest[DIGEST_LEN];
static char geoip6_digest[DIGEST_LEN];
+/* Total size in bytes of the geoip client history cache. Used by the OOM
+ * handler. */
+static size_t geoip_client_history_cache_size;
+
/** Return the index of the <b>country</b>'s entry in the GeoIP
* country list if it is a valid 2-letter country code, otherwise
* return -1. */
@@ -526,6 +530,15 @@ HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+/** Return the size of a client map entry. */
+static inline size_t
+clientmap_entry_size(const clientmap_entry_t *ent)
+{
+ tor_assert(ent);
+ return (sizeof(clientmap_entry_t) +
+ (ent->transport_name ? strlen(ent->transport_name) : 0));
+}
+
/** Free all storage held by <b>ent</b>. */
static void
clientmap_entry_free(clientmap_entry_t *ent)
@@ -533,6 +546,8 @@ clientmap_entry_free(clientmap_entry_t *ent)
if (!ent)
return;
+ geoip_client_history_cache_size -= clientmap_entry_size(ent);
+
tor_free(ent->transport_name);
tor_free(ent);
}
@@ -595,6 +610,7 @@ geoip_note_client_seen(geoip_client_action_t action,
ent->transport_name = tor_strdup(transport_name);
ent->action = (int)action;
HT_INSERT(clientmap, &client_history, ent);
+ geoip_client_history_cache_size += clientmap_entry_size(ent);
}
if (now / 60 <= (int)MAX_LAST_SEEN_IN_MINUTES && now >= 0)
ent->last_seen_in_minutes = (unsigned)(now/60);
@@ -635,6 +651,81 @@ geoip_remove_old_clients(time_t cutoff)
&cutoff);
}
+/* Cleanup client entries older than the cutoff. Used for the OOM. Return the
+ * number of bytes freed. If 0 is returned, nothing was freed. */
+static size_t
+oom_clean_client_entries(time_t cutoff)
+{
+ size_t bytes = 0;
+ clientmap_entry_t **ent, **ent_next;
+
+ for (ent = HT_START(clientmap, &client_history); ent; ent = ent_next) {
+ clientmap_entry_t *entry = *ent;
+ if (entry->last_seen_in_minutes < (cutoff / 60)) {
+ ent_next = HT_NEXT_RMV(clientmap, &client_history, ent);
+ bytes += clientmap_entry_size(entry);
+ clientmap_entry_free(entry);
+ } else {
+ ent_next = HT_NEXT(clientmap, &client_history, ent);
+ }
+ }
+ return bytes;
+}
+
+/* Below this minimum lifetime, the OOM won't cleanup any entries. */
+#define GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF (4 * 60 * 60)
+/* The OOM moves the cutoff by that much every run. */
+#define GEOIP_CLIENT_CACHE_OOM_STEP (15 * 50)
+
+/* Cleanup the geoip client history cache called from the OOM handler. Return
+ * the amount of bytes removed. This can return a value below or above
+ * min_remove_bytes but will stop as oon as the min_remove_bytes has been
+ * reached. */
+size_t
+geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes)
+{
+ time_t k;
+ size_t bytes_removed = 0;
+
+ /* Our OOM handler called with 0 bytes to remove is a code flow error. */
+ tor_assert(min_remove_bytes != 0);
+
+ /* Set k to the initial cutoff of an entry. We then going to move it by step
+ * to try to remove as much as we can. */
+ k = WRITE_STATS_INTERVAL;
+
+ do {
+ time_t cutoff;
+
+ /* If k has reached the minimum lifetime, we have to stop else we might
+ * remove every single entries which would be pretty bad for the DoS
+ * mitigation subsystem if by just filling the geoip cache, it was enough
+ * to trigger the OOM and clean every single entries. */
+ if (k <= GEOIP_CLIENT_CACHE_OOM_MIN_CUTOFF) {
+ break;
+ }
+
+ cutoff = now - k;
+ bytes_removed += oom_clean_client_entries(cutoff);
+ k -= GEOIP_CLIENT_CACHE_OOM_STEP;
+ } while (bytes_removed < min_remove_bytes);
+
+ return bytes_removed;
+}
+
+/* Return the total size in bytes of the client history cache. */
+size_t
+geoip_client_cache_total_allocation(void)
+{
+ size_t bytes = 0;
+ clientmap_entry_t **ent;
+
+ HT_FOREACH(ent, clientmap, &client_history) {
+ bytes += clientmap_entry_size(*ent);
+ }
+ return bytes;
+}
+
/** How many responses are we giving to clients requesting v3 network
* statuses? */
static uint32_t ns_v3_responses[GEOIP_NS_RESPONSE_NUM];
diff --git a/src/or/geoip.h b/src/or/geoip.h
index 070296dd0..42d0c1cfd 100644
--- a/src/or/geoip.h
+++ b/src/or/geoip.h
@@ -33,6 +33,8 @@ void geoip_note_client_seen(geoip_client_action_t action,
const tor_addr_t *addr, const char *transport_name,
time_t now);
void geoip_remove_old_clients(time_t cutoff);
+size_t geoip_client_cache_total_allocation(void);
+size_t geoip_client_cache_handle_oom(time_t now, size_t min_remove_bytes);
void geoip_note_ns_response(geoip_ns_response_t response);
char *geoip_get_transport_history(void);
diff --git a/src/or/relay.c b/src/or/relay.c
index 29f34ca03..22ce76752 100644
--- a/src/or/relay.c
+++ b/src/or/relay.c
@@ -2469,24 +2469,34 @@ static time_t last_time_under_memory_pressure = 0;
STATIC int
cell_queues_check_size(void)
{
+ time_t now = time(NULL);
size_t alloc = cell_queues_get_total_allocation();
alloc += buf_get_total_allocation();
alloc += tor_zlib_get_total_allocation();
const size_t rend_cache_total = rend_cache_get_total_allocation();
alloc += rend_cache_total;
+ const size_t geoip_client_cache_total =
+ geoip_client_cache_total_allocation();
+ alloc += geoip_client_cache_total;
if (alloc >= get_options()->MaxMemInQueues_low_threshold) {
last_time_under_memory_pressure = approx_time();
if (alloc >= get_options()->MaxMemInQueues) {
/* If we're spending over 20% of the memory limit on hidden service
- * descriptors, free them until we're down to 10%.
- */
+ * descriptors, free them until we're down to 10%. Do the same for geoip
+ * client cache. */
if (rend_cache_total > get_options()->MaxMemInQueues / 5) {
const size_t bytes_to_remove =
rend_cache_total - (size_t)(get_options()->MaxMemInQueues / 10);
- rend_cache_clean_v2_descs_as_dir(time(NULL), bytes_to_remove);
+ rend_cache_clean_v2_descs_as_dir(now, bytes_to_remove);
alloc -= rend_cache_total;
alloc += rend_cache_get_total_allocation();
}
+ if (geoip_client_cache_total > get_options()->MaxMemInQueues / 5) {
+ const size_t bytes_to_remove =
+ geoip_client_cache_total -
+ (size_t)(get_options()->MaxMemInQueues / 10);
+ alloc -= geoip_client_cache_handle_oom(now, bytes_to_remove);
+ }
circuits_handle_oom(alloc);
return 1;
}
diff --git a/src/test/test.c b/src/test/test.c
index 9a41b976b..e2fbfd21b 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -912,6 +912,24 @@ test_geoip(void *arg)
tt_str_op(entry_stats_2,OP_EQ, s);
tor_free(s);
+ /* Test the OOM handler. Add a client, run the OOM. */
+ geoip_entry_stats_init(now);
+ SET_TEST_ADDRESS(100);
+ geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+ now - (12 * 60 * 60));
+ /* We've seen this 12 hours ago. Run the OOM, it should clean the entry
+ * because it is above the minimum cutoff of 4 hours. */
+ size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+ tt_size_op(bytes_removed, OP_GT, 0);
+
+ /* Do it again but this time with an entry with a lower cutoff. */
+ geoip_entry_stats_init(now);
+ SET_TEST_ADDRESS(100);
+ geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL,
+ now - (3 * 60 * 60));
+ bytes_removed = geoip_client_cache_handle_oom(now, 1000);
+ tt_size_op(bytes_removed, OP_EQ, 0);
+
/* Stop collecting entry statistics. */
geoip_entry_stats_term();
get_options_mutable()->EntryStatistics = 0;
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits