[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] Next batch of memory miserdom: mmap cached-routers file. T...
Update of /home/or/cvsroot/tor/src/or
In directory moria:/tmp/cvs-serv564/src/or
Modified Files:
control.c directory.c dirserv.c or.h router.c routerlist.c
routerparse.c test.c
Log Message:
Next batch of memory miserdom: mmap cached-routers file. This is sure to break somewhere.
Index: control.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/control.c,v
retrieving revision 1.194
retrieving revision 1.195
diff -u -p -d -r1.194 -r1.195
--- control.c 13 Jun 2006 09:16:09 -0000 1.194
+++ control.c 22 Jun 2006 07:01:54 -0000 1.195
@@ -1330,14 +1330,14 @@ handle_getinfo_helper(const char *questi
if (ri) {
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
- *answer = tor_strdup(body);
+ *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
}
} else if (!strcmpstart(question, "desc/name/")) {
routerinfo_t *ri = router_get_by_nickname(question+strlen("desc/name/"),1);
if (ri) {
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
- *answer = tor_strdup(body);
+ *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
}
} else if (!strcmp(question, "desc/all-recent")) {
routerlist_t *routerlist = router_get_routerlist();
@@ -1347,7 +1347,8 @@ handle_getinfo_helper(const char *questi
{
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
- smartlist_add(sl, tor_strdup(body));
+ smartlist_add(sl,
+ tor_strndup(body, ri->cache_info.signed_descriptor_len));
});
}
*answer = smartlist_join_strings(sl, "", 0, NULL);
Index: directory.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/directory.c,v
retrieving revision 1.382
retrieving revision 1.383
diff -u -p -d -r1.382 -r1.383
--- directory.c 21 Jun 2006 04:57:12 -0000 1.382
+++ directory.c 22 Jun 2006 07:01:54 -0000 1.383
@@ -1076,7 +1076,7 @@ connection_dir_client_reached_eof(connec
if (which || (conn->requested_resource &&
!strcmpstart(conn->requested_resource, "all"))) {
/* as we learn from them, we remove them from 'which' */
- router_load_routers_from_string(body, 0, which);
+ router_load_routers_from_string(body, SAVED_NOWHERE, which);
directory_info_has_arrived(time(NULL), 0);
}
if (which) { /* mark remaining ones as failed */
Index: dirserv.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/dirserv.c,v
retrieving revision 1.348
retrieving revision 1.349
diff -u -p -d -r1.348 -r1.349
--- dirserv.c 21 Jun 2006 04:57:12 -0000 1.348
+++ dirserv.c 22 Jun 2006 07:01:54 -0000 1.349
@@ -481,7 +481,7 @@ dirserv_add_descriptor(const char *desc,
*msg = NULL;
/* Check: is the descriptor syntactically valid? */
- ri = router_parse_entry_from_string(desc, NULL);
+ ri = router_parse_entry_from_string(desc, NULL, 1);
if (!ri) {
log_warn(LD_DIRSERV, "Couldn't parse uploaded server descriptor");
*msg = "Rejected: Couldn't parse server descriptor.";
@@ -1808,6 +1808,7 @@ connection_dirserv_add_servers_to_outbuf
while (smartlist_len(conn->fingerprint_stack) &&
buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
+ char *body;
char *fp = smartlist_pop_last(conn->fingerprint_stack);
signed_descriptor_t *sd = NULL;
if (by_fp) {
@@ -1824,13 +1825,14 @@ connection_dirserv_add_servers_to_outbuf
tor_free(fp);
if (!sd)
continue;
+ body = signed_descriptor_get_body(sd);
if (conn->zlib_state) {
connection_write_to_buf_zlib(
conn, conn->zlib_state,
- sd->signed_descriptor_body, sd->signed_descriptor_len,
+ body, sd->signed_descriptor_len,
0);
} else {
- connection_write_to_buf(sd->signed_descriptor_body,
+ connection_write_to_buf(body,
sd->signed_descriptor_len,
conn);
}
Index: or.h
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/or.h,v
retrieving revision 1.845
retrieving revision 1.846
diff -u -p -d -r1.845 -r1.846
--- or.h 21 Jun 2006 04:57:12 -0000 1.845
+++ or.h 22 Jun 2006 07:01:54 -0000 1.846
@@ -762,6 +762,10 @@ typedef struct cached_dir_t {
int refcnt; /**< Reference count for this cached_dir_t. */
} cached_dir_t;
+typedef enum {
+ SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL
+} saved_location_t;
+
/** Information need to cache an onion router's descriptor. */
typedef struct signed_descriptor_t {
char *signed_descriptor_body;
@@ -769,7 +773,7 @@ typedef struct signed_descriptor_t {
char signed_descriptor_digest[DIGEST_LEN];
char identity_digest[DIGEST_LEN];
time_t published_on;
- enum { SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL } saved_location;
+ saved_location_t saved_location;
off_t saved_offset;
} signed_descriptor_t;
@@ -933,6 +937,9 @@ typedef struct {
/** List of signed_descriptor_t for older router descriptors we're
* caching. */
smartlist_t *old_routers;
+ /** DOCDOC */
+ const char *mmap_descriptors;
+ size_t mmap_descriptors_len;
} routerlist_t;
/** Information on router used when extending a circuit. (We don't need a
@@ -2367,7 +2374,8 @@ int router_add_to_routerlist(routerinfo_
int from_cache, int from_fetch);
int router_load_single_router(const char *s, uint8_t purpose,
const char **msg);
-void router_load_routers_from_string(const char *s, int from_cache,
+void router_load_routers_from_string(const char *s,
+ saved_location_t saved_location,
smartlist_t *requested_fingerprints);
typedef enum {
NS_FROM_CACHE, NS_FROM_DIR, NS_GENERATED
@@ -2440,7 +2448,7 @@ int router_append_dirobj_signature(char
crypto_pk_env_t *private_key);
int router_parse_list_from_string(const char **s,
smartlist_t *dest,
- int from_cache);
+ saved_location_t saved_location);
int router_parse_routerlist_from_directory(const char *s,
routerlist_t **dest,
crypto_pk_env_t *pkey,
@@ -2448,7 +2456,8 @@ int router_parse_routerlist_from_directo
int write_to_cache);
int router_parse_runningrouters(const char *str);
int router_parse_directory(const char *str);
-routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
+routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
+ int cache_copy);
addr_policy_t *router_parse_addr_policy_from_string(const char *s,
int assume_action);
version_status_t tor_version_is_obsolete(const char *myversion,
Index: router.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/router.c,v
retrieving revision 1.266
retrieving revision 1.267
diff -u -p -d -r1.266 -r1.267
--- router.c 13 Jun 2006 12:05:59 -0000 1.266
+++ router.c 22 Jun 2006 07:01:54 -0000 1.267
@@ -777,7 +777,10 @@ router_get_my_descriptor(void)
const char *body;
if (!router_get_my_routerinfo())
return NULL;
+ /* Make sure this is nul-terminated. */
+ tor_assert(desc_routerinfo->cache_info.saved_location == SAVED_NOWHERE);
body = signed_descriptor_get_body(&desc_routerinfo->cache_info);
+ tor_assert(!body[desc_routerinfo->cache_info.signed_descriptor_len]);
log_debug(LD_GENERAL,"my desc is '%s'", body);
return body;
}
@@ -1187,7 +1190,7 @@ router_dump_router_to_string(char *s, si
#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
cp = s_tmp = s_dup = tor_strdup(s);
- ri_tmp = router_parse_entry_from_string(cp, NULL);
+ ri_tmp = router_parse_entry_from_string(cp, NULL, 1);
if (!ri_tmp) {
log_err(LD_BUG,
"We just generated a router descriptor we can't parse: <<%s>>",
Index: routerlist.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/routerlist.c,v
retrieving revision 1.521
retrieving revision 1.522
diff -u -p -d -r1.521 -r1.522
--- routerlist.c 21 Jun 2006 04:57:12 -0000 1.521
+++ routerlist.c 22 Jun 2006 07:01:54 -0000 1.522
@@ -194,6 +194,21 @@ router_append_to_journal(signed_descript
return 0;
}
+static int
+_compare_old_routers_by_age(const void **_a, const void **_b)
+{
+ const signed_descriptor_t *r1 = *_a, *r2 = *_b;
+ return r1->published_on - r2->published_on;
+}
+
+static int
+_compare_routers_by_age(const void **_a, const void **_b)
+{
+ const routerinfo_t *r1 = *_a, *r2 = *_b;
+ return r1->cache_info.published_on - r2->cache_info.published_on;
+}
+
+
/** If the journal is too long, or if <b>force</b> is true, then atomically
* replace the router store with the routers currently in our routerlist, and
* clear the journal. Return 0 on success, -1 on failure.
@@ -223,8 +238,16 @@ router_rebuild_store(int force)
chunk_list = smartlist_create();
for (i = 0; i < 2; ++i) {
- smartlist_t *lst = (i == 0) ? routerlist->old_routers :
- routerlist->routers;
+ smartlist_t *lst = smartlist_create();
+ /* We sort the routers by age to enhance locality on disk. */
+ if (i==0) {
+ smartlist_add_all(lst, routerlist->old_routers);
+ smartlist_sort(lst, _compare_old_routers_by_age);
+ } else {
+ smartlist_add_all(lst, routerlist->routers);
+ smartlist_sort(lst, _compare_routers_by_age);
+ }
+ /* Now, add the appropriate members to chunk_list */
SMARTLIST_FOREACH(lst, void *, ptr,
{
signed_descriptor_t *sd = (i==0) ?
@@ -233,6 +256,7 @@ router_rebuild_store(int force)
const char *body = signed_descriptor_get_body(sd);
if (!body) {
log_warn(LD_BUG, "Bug! No descriptor available for router.");
+ smartlist_free(lst);
goto done;
}
c = tor_malloc(sizeof(sized_chunk_t));
@@ -240,11 +264,22 @@ router_rebuild_store(int force)
c->len = sd->signed_descriptor_len;
smartlist_add(chunk_list, c);
});
+ smartlist_free(lst);
}
if (write_chunks_to_file(fname, chunk_list, 0)<0) {
log_warn(LD_FS, "Error writing router store to disk.");
goto done;
}
+ /* Our mmap is now invalid. */
+ if (routerlist->mmap_descriptors) {
+ tor_munmap_file(routerlist->mmap_descriptors,
+ routerlist->mmap_descriptors_len);
+ routerlist->mmap_descriptors =
+ tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
+ if (! routerlist->mmap_descriptors)
+ log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
+ }
+
for (i = 0; i < 2; ++i) {
smartlist_t *lst = (i == 0) ? routerlist->old_routers :
routerlist->routers;
@@ -256,6 +291,8 @@ router_rebuild_store(int force)
sd->saved_location = SAVED_IN_CACHE;
sd->saved_offset = offset;
+ if (routerlist->mmap_descriptors)
+ sd->signed_descriptor_body = NULL;
offset += sd->signed_descriptor_len;
});
}
@@ -286,31 +323,32 @@ router_reload_router_list(void)
{
or_options_t *options = get_options();
size_t fname_len = strlen(options->DataDirectory)+32;
- char *fname = tor_malloc(fname_len);
+ char *fname = tor_malloc(fname_len), *contents;
struct stat st;
- int j;
if (!routerlist)
router_get_routerlist(); /* mallocs and inits it in place */
router_journal_len = router_store_len = 0;
- for (j = 0; j < 2; ++j) {
- char *contents;
- tor_snprintf(fname, fname_len,
- (j==0)?"%s/cached-routers":"%s/cached-routers.new",
- options->DataDirectory);
- contents = read_file_to_str(fname, 0);
- if (contents) {
- stat(fname, &st);
- if (j==0)
- router_store_len = st.st_size;
- else
- router_journal_len = st.st_size;
- router_load_routers_from_string(contents, 1, NULL);
- tor_free(contents);
- }
+ tor_snprintf(fname, fname_len, "%s/cached-routers", options->DataDirectory);
+ routerlist->mmap_descriptors =
+ tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
+ if (routerlist->mmap_descriptors) {
+ router_store_len = routerlist->mmap_descriptors_len;
+ router_load_routers_from_string(routerlist->mmap_descriptors,
+ SAVED_IN_CACHE, NULL);
}
+
+ tor_snprintf(fname, fname_len, "%s/cached-routers.new",
+ options->DataDirectory);
+ contents = read_file_to_str(fname, 0);
+ if (contents) {
+ stat(fname, &st);
+ router_load_routers_from_string(contents,
+ SAVED_IN_JOURNAL, NULL);
+ }
+
tor_free(fname);
if (router_journal_len) {
@@ -1103,10 +1141,26 @@ router_get_by_descriptor_digest(const ch
return digestmap_get(routerlist->desc_digest_map, digest);
}
+/* DOCDOC Not always nul-terminated. */
const char *
signed_descriptor_get_body(signed_descriptor_t *desc)
{
- return desc->signed_descriptor_body;
+ const char *r;
+ size_t len = desc->signed_descriptor_len;
+ tor_assert(len > 32);
+ if (desc->saved_location == SAVED_IN_CACHE && routerlist &&
+ routerlist->mmap_descriptors) {
+ tor_assert(desc->saved_offset + len <= routerlist->mmap_descriptors_len);
+ r = routerlist->mmap_descriptors + desc->saved_offset;
+ } else {
+ r = desc->signed_descriptor_body;
+ }
+ tor_assert(r);
+ tor_assert(!memcmp("router ", r, 7));
+ tor_assert(!memcmp("\n-----END SIGNATURE-----\n",
+ r + len - 25, 25));
+
+ return r;
}
/** Return the current list of all known routers. */
@@ -1887,7 +1941,7 @@ router_load_single_router(const char *s,
tor_assert(msg);
*msg = NULL;
- if (!(ri = router_parse_entry_from_string(s, NULL))) {
+ if (!(ri = router_parse_entry_from_string(s, NULL, 1))) {
log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
*msg = "Couldn't parse router descriptor.";
return -1;
@@ -1927,16 +1981,18 @@ router_load_single_router(const char *s,
* uppercased identity fingerprints. Do not update any router whose
* fingerprint is not on the list; after updating a router, remove its
* fingerprint from the list.
+ * DOCDOC saved_location
*/
void
-router_load_routers_from_string(const char *s, int from_cache,
+router_load_routers_from_string(const char *s, saved_location_t saved_location,
smartlist_t *requested_fingerprints)
{
smartlist_t *routers = smartlist_create(), *changed = smartlist_create();
char fp[HEX_DIGEST_LEN+1];
const char *msg;
+ int from_cache = (saved_location != SAVED_NOWHERE);
- router_parse_list_from_string(&s, routers, from_cache);
+ router_parse_list_from_string(&s, routers, saved_location);
routers_update_status_from_networkstatus(routers, !from_cache);
Index: routerparse.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/routerparse.c,v
retrieving revision 1.185
retrieving revision 1.186
diff -u -p -d -r1.185 -r1.186
--- routerparse.c 29 Apr 2006 18:42:26 -0000 1.185
+++ routerparse.c 22 Jun 2006 07:01:54 -0000 1.186
@@ -635,10 +635,11 @@ check_directory_signature(const char *di
* are marked running and valid. Advances *s to a point immediately
* following the last router entry. Ignore any trailing router entries that
* are not complete. Returns 0 on success and -1 on failure.
+ * DOCDOC saved_location
*/
int
router_parse_list_from_string(const char **s, smartlist_t *dest,
- int from_cache)
+ saved_location_t saved_location)
{
routerinfo_t *router;
const char *end, *cp, *start;
@@ -678,15 +679,16 @@ router_parse_list_from_string(const char
continue;
}
- router = router_parse_entry_from_string(*s, end);
+ router = router_parse_entry_from_string(*s, end,
+ saved_location != SAVED_IN_CACHE);
*s = end;
if (!router) {
log_warn(LD_DIR, "Error reading router; skipping");
continue;
}
- if (from_cache) {
- router->cache_info.saved_location = SAVED_IN_CACHE;
+ if (saved_location != SAVED_NOWHERE) {
+ router->cache_info.saved_location = saved_location;
router->cache_info.saved_offset = *s - start;
}
smartlist_add(dest, router);
@@ -698,9 +700,11 @@ router_parse_list_from_string(const char
/** Helper function: reads a single router entry from *<b>s</b> ...
* *<b>end</b>. Mallocs a new router and returns it if all goes well, else
* returns NULL.
+ * DOCDOC cache_copy
*/
routerinfo_t *
-router_parse_entry_from_string(const char *s, const char *end)
+router_parse_entry_from_string(const char *s, const char *end,
+ int cache_copy)
{
routerinfo_t *router = NULL;
char signed_digest[128];
@@ -747,7 +751,8 @@ router_parse_entry_from_string(const cha
}
router = tor_malloc_zero(sizeof(routerinfo_t));
- router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
+ if (cache_copy)
+ router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
router->cache_info.signed_descriptor_len = end-s;
memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
Index: test.c
===================================================================
RCS file: /home/or/cvsroot/tor/src/or/test.c,v
retrieving revision 1.228
retrieving revision 1.229
diff -u -p -d -r1.228 -r1.229
--- test.c 18 Jun 2006 07:35:10 -0000 1.228
+++ test.c 22 Jun 2006 07:01:54 -0000 1.229
@@ -1297,7 +1297,7 @@ test_dir_format(void)
test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0);
cp = buf;
- rp1 = router_parse_entry_from_string((const char*)cp,NULL);
+ rp1 = router_parse_entry_from_string((const char*)cp,NULL,1);
test_assert(rp1);
test_streq(rp1->address, r1.address);
test_eq(rp1->or_port, r1.or_port);
@@ -1320,7 +1320,7 @@ test_dir_format(void)
test_streq(buf, buf2);
cp = buf;
- rp2 = router_parse_entry_from_string(&cp);
+ rp2 = router_parse_entry_from_string(&cp,1);
test_assert(rp2);
test_streq(rp2->address, r2.address);
test_eq(rp2->or_port, r2.or_port);