[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r10004: When logging memory usage, break down memory used in buffers (in tor/trunk: . src/or)

To: or-cvs@xxxxxxxxxxxxx
Subject: [or-cvs] r10004: When logging memory usage, break down memory used in buffers (in tor/trunk: . src/or)
From: nickm@xxxxxxxx
Date: Sun, 22 Apr 2007 23:04:48 -0400 (EDT)
Delivered-to: archiver@seul.org
Delivered-to: or-cvs-outgoing@seul.org
Delivered-to: or-cvs@seul.org
Delivery-date: Sun, 22 Apr 2007 23:04:58 -0400
Reply-to: or-talk@xxxxxxxxxxxxx
Sender: owner-or-cvs@xxxxxxxxxxxxx

Author: nickm
Date: 2007-04-22 23:04:46 -0400 (Sun, 22 Apr 2007)
New Revision: 10004

Modified:
   tor/trunk/
   tor/trunk/ChangeLog
   tor/trunk/src/or/buffers.c
   tor/trunk/src/or/connection.c
   tor/trunk/src/or/main.c
   tor/trunk/src/or/or.h
Log:
 r12496@catbus:  nickm | 2007-04-22 23:04:05 -0400
 When logging memory usage, break down memory used in buffers by buffer type.



Property changes on: tor/trunk
___________________________________________________________________
 svk:merge ticket from /tor/trunk [r12496] on 8246c3cf-6607-4228-993b-4d95d33730f1

Modified: tor/trunk/ChangeLog
===================================================================
--- tor/trunk/ChangeLog	2007-04-23 00:24:06 UTC (rev 10003)
+++ tor/trunk/ChangeLog	2007-04-23 03:04:46 UTC (rev 10004)
@@ -43,6 +43,8 @@
     - Put a platform string (e.g. "Linux i686") in the startup log
       message, so when people paste just their logs, we know if it's
       openbsd or windows or what.
+    - When logging memory usage, break down memory used in buffers by
+      buffer type.
 
   o Minor features (directory system):
     - Directory authorities accept and serve "extra info" documents for

Modified: tor/trunk/src/or/buffers.c
===================================================================
--- tor/trunk/src/or/buffers.c	2007-04-23 00:24:06 UTC (rev 10003)
+++ tor/trunk/src/or/buffers.c	2007-04-23 03:04:46 UTC (rev 10004)
@@ -75,11 +75,6 @@
   size_t datalen; /**< Number of bytes currently in <b>mem</b>. */
 };
 
-/** How many bytes, total, are used in all buffers? */
-uint64_t buf_total_used = 0;
-/** How many bytes, total, are allocated in all buffers? */
-uint64_t buf_total_alloc = 0;
-
 /** Size, in bytes, for newly allocated buffers.  Should be a power of 2. */
 #define INITIAL_BUF_SIZE (4*1024)
 /** Size, in bytes, for minimum 'shrink' size for buffers.  Buffers may start
@@ -228,8 +223,6 @@
     SET_GUARDS(buf->mem, new_capacity);
     buf->cur = buf->mem+offset;
   }
-  buf_total_alloc += new_capacity;
-  buf_total_alloc -= buf->len;
 
   if (offset + buf->datalen > buf->len) {
     /* We need to move data now that we are done growing.  The buffer
@@ -317,7 +310,6 @@
 {
   tor_assert(buf->datalen >= n);
   buf->datalen -= n;
-  buf_total_used -= n;
   if (buf->datalen) {
     buf->cur = _wrap_ptr(buf, buf->cur+n);
   } else {
@@ -347,7 +339,6 @@
   SET_GUARDS(buf->mem, size);
   buf->len = buf->memsize = size;
 
-  buf_total_alloc += size;
   assert_buf_ok(buf);
   return buf;
 }
@@ -363,7 +354,6 @@
 void
 buf_clear(buf_t *buf)
 {
-  buf_total_used -= buf->datalen;
   buf->datalen = 0;
   buf->cur = buf->mem;
   buf->len = buf->memsize;
@@ -401,8 +391,6 @@
   buf->magic = 0xDEADBEEF;
   oldmem = RAW_MEM(buf->mem);
   tor_free(oldmem);
-  buf_total_alloc -= buf->len;
-  buf_total_used -= buf->datalen;
   tor_free(buf);
 }
 
@@ -436,7 +424,6 @@
     return 0;
   } else { /* we read some bytes */
     buf->datalen += read_result;
-    buf_total_used += read_result;
     if (buf->datalen > buf->highwater)
       buf->highwater = buf->datalen;
     log_debug(LD_NET,"Read %d bytes. %d on inbuf.",read_result,
@@ -512,7 +499,6 @@
   if (r<0)
     return r;
   buf->datalen += r;
-  buf_total_used += r;
   if (buf->datalen > buf->highwater)
     buf->highwater = buf->datalen;
   log_debug(LD_NET,"Read %d bytes. %d on inbuf; %d pending",r,
@@ -756,13 +742,11 @@
 
   memcpy(next, string, string_len);
   buf->datalen += string_len;
-  buf_total_used += string_len;
 
   if (len2) {
     tor_assert(_buf_end(buf) == buf->mem);
     memcpy(buf->mem, string+string_len, len2);
     buf->datalen += len2;
-    buf_total_used += len2;
   }
   if (buf->datalen > buf->highwater)
     buf->highwater = buf->datalen;
@@ -1410,7 +1394,6 @@
     buf->datalen += old_avail - avail;
     if (buf->datalen > buf->highwater)
       buf->highwater = buf->datalen;
-    buf_total_used += old_avail - avail;
   } while (!over);
   return 0;
 }

Modified: tor/trunk/src/or/connection.c
===================================================================
--- tor/trunk/src/or/connection.c	2007-04-23 00:24:06 UTC (rev 10003)
+++ tor/trunk/src/or/connection.c	2007-04-23 03:04:46 UTC (rev 10004)
@@ -389,7 +389,6 @@
   connection_t **carray;
 
   get_connection_array(&carray,&n);
-
   /* We don't want to log any messages to controllers. */
   for (i=0;i<n;i++)
     if (carray[i]->type == CONN_TYPE_CONTROL)
@@ -2503,6 +2502,55 @@
   }
 }
 
+/** Log how many bytes are used by buffers of different kinds and sizes. */
+void
+connection_dump_buffer_mem_stats(int severity)
+{
+  uint64_t used_by_type[_CONN_TYPE_MAX+1];
+  uint64_t alloc_by_type[_CONN_TYPE_MAX+1];
+  int n_conns_by_type[_CONN_TYPE_MAX+1];
+  uint64_t total_alloc = 0;
+  uint64_t total_used = 0;
+  int i, n;
+  connection_t **carray;
+
+  memset(used_by_type, 0, sizeof(used_by_type));
+  memset(alloc_by_type, 0, sizeof(alloc_by_type));
+  memset(n_conns_by_type, 0, sizeof(n_conns_by_type));
+
+  get_connection_array(&carray,&n);
+
+  for (i=0; i<n; ++i) {
+    connection_t *c = carray[i];
+    int tp = c->type;
+    ++n_conns_by_type[tp];
+    if (c->inbuf) {
+      used_by_type[tp] += buf_datalen(c->inbuf);
+      alloc_by_type[tp] += buf_capacity(c->inbuf);
+    }
+    if (c->outbuf) {
+      used_by_type[tp] += buf_datalen(c->outbuf);
+      alloc_by_type[tp] += buf_capacity(c->outbuf);
+    }
+  }
+  for (i=0; i <= _CONN_TYPE_MAX; ++i) {
+    total_used += used_by_type[i];
+    total_alloc += alloc_by_type[i];
+  }
+
+  log(severity, LD_GENERAL,
+     "In buffers for %d connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+      n, U64_PRINTF_ARG(total_used), U64_PRINTF_ARG(total_alloc));
+  for (i=_CONN_TYPE_MIN; i <= _CONN_TYPE_MAX; ++i) {
+    if (!n_conns_by_type[i])
+      continue;
+    log(severity, LD_GENERAL,
+        "  For %d %s connections: "U64_FORMAT" used/"U64_FORMAT" allocated",
+        n_conns_by_type[i], conn_type_to_string(i),
+        U64_PRINTF_ARG(used_by_type[i]), U64_PRINTF_ARG(alloc_by_type[i]));
+  }
+}
+
 /** Verify that connection <b>conn</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */

Modified: tor/trunk/src/or/main.c
===================================================================
--- tor/trunk/src/or/main.c	2007-04-23 00:24:06 UTC (rev 10003)
+++ tor/trunk/src/or/main.c	2007-04-23 03:04:46 UTC (rev 10004)
@@ -1533,8 +1533,6 @@
   }
 }
 
-extern uint64_t buf_total_used;
-extern uint64_t buf_total_alloc;
 extern uint64_t rephist_total_alloc;
 extern uint32_t rephist_total_num;
 
@@ -1544,10 +1542,7 @@
 static void
 dumpmemusage(int severity)
 {
-  log(severity, LD_GENERAL,
-      "In buffers: "U64_FORMAT" used/"U64_FORMAT" allocated (%d conns).",
-      U64_PRINTF_ARG(buf_total_used), U64_PRINTF_ARG(buf_total_alloc),
-      n_conns);
+  connection_dump_buffer_mem_stats(severity);
   log(severity, LD_GENERAL, "In rephist: "U64_FORMAT" used by %d Tors.",
       U64_PRINTF_ARG(rephist_total_alloc), rephist_total_num);
   dump_routerlist_mem_usage(severity);

Modified: tor/trunk/src/or/or.h
===================================================================
--- tor/trunk/src/or/or.h	2007-04-23 00:24:06 UTC (rev 10003)
+++ tor/trunk/src/or/or.h	2007-04-23 03:04:46 UTC (rev 10004)
@@ -2252,6 +2252,7 @@
 
 void assert_connection_ok(connection_t *conn, time_t now);
 int connection_or_nonopen_was_started_here(or_connection_t *conn);
+void connection_dump_buffer_mem_stats(int severity);
 
 /********************************* connection_edge.c *************************/

Prev by Author: [or-cvs] r10003: Even more asserts to try to catch bug 404/417. (in tor/trunk: . src/or)
Next by Author: [or-cvs] r10006: Keep a freelist of unused 4k buffer chunks, rather than wast (in tor/trunk: . src/common src/or)
Previous by thread: [or-cvs] r10003: Even more asserts to try to catch bug 404/417. (in tor/trunk: . src/or)
Next by thread: [or-cvs] r10005: Include the kqueue update in this branch as well. (tor/trunk/contrib/osx)
Index(es):
- Author
- Thread