[or-cvs] r8761: Never discard a descriptor for being too old until either it (in tor/trunk: . doc src/or)

[nickm@xxxxxxxx]

Author: nickm
Date: 2006-10-19 19:04:56 -0400 (Thu, 19 Oct 2006)
New Revision: 8761

Modified:
   tor/trunk/
   tor/trunk/ChangeLog
   tor/trunk/doc/TODO
   tor/trunk/doc/dir-spec.txt
   tor/trunk/src/or/or.h
   tor/trunk/src/or/routerlist.c
Log:
 r9273@Kushana:  nickm | 2006-10-19 15:43:39 -0400
 Never discard a descriptor for being too old until either it is recommended by no authorities, or until we download a better (more recent and recommended) one for the same router. This will eventually make it possible for servers to publish less often.



Property changes on: tor/trunk
___________________________________________________________________
 svk:merge ticket from /tor/trunk [r9273] on c95137ef-5f19-0410-b913-86e773d04f59

Modified: tor/trunk/ChangeLog
===================================================================
--- tor/trunk/ChangeLog	2006-10-19 23:04:49 UTC (rev 8760)
+++ tor/trunk/ChangeLog	2006-10-19 23:04:56 UTC (rev 8761)
@@ -3,6 +3,11 @@
     - If most authorities set a (newly defined) BadExit flag for a server,
       do not consider it as a general-purpose exit.  Only consider
       authorities that advertise themselves as listing bad exits.
+    - Start making it possible for servers to publish less often: never
+      discard a descriptor simply for being too old until either it is
+      recommended by no authorities, or until we get a better one for the
+      same router.  Make caches consider retaining old recommended
+      routers for even longer.
 
   o Minor features, controller:
     - Add a REASON field to CIRC events; for backward compatibility, this

Modified: tor/trunk/doc/TODO
===================================================================
--- tor/trunk/doc/TODO	2006-10-19 23:04:49 UTC (rev 8760)
+++ tor/trunk/doc/TODO	2006-10-19 23:04:56 UTC (rev 8761)
@@ -142,8 +142,8 @@
 x     - spec
 d     - implement
 
-  - A more efficient dir protocol.
-N   - Later, servers will stop generating new descriptors simply
+  o Changes towards a more efficient dir protocol.
+    o Later, servers will stop generating new descriptors simply
       because 18 hours have passed: we must start tolerating this now.
 
   - Critical but minor bugs, backport candidates.

Modified: tor/trunk/doc/dir-spec.txt
===================================================================
--- tor/trunk/doc/dir-spec.txt	2006-10-19 23:04:49 UTC (rev 8760)
+++ tor/trunk/doc/dir-spec.txt	2006-10-19 23:04:56 UTC (rev 8761)
@@ -628,7 +628,7 @@
    from an authority, they should not need to go to the authority directly
    again.)
 
-5.2. Downloading router descriptors
+5.2. Downloading and storing router descriptors
 
    Clients try to have the best descriptor for each router.  A descriptor is
    "best" if:
@@ -665,6 +665,12 @@
    No descriptors are downloaded until the client has downloaded more than
    half of the network-status documents.
 
+   Clients retain the most recent descriptor they have downloaded for each
+   router so long as it is not too old (currently, 48 hours), OR so long as
+   it is recommended by at least one networkstatus AND no "better"
+   descriptor has been downloaded.  [Versions of Tor before 0.1.2.3-alpha
+   would discard descriptors simply for being published too far in the past.]
+
 5.3. Managing downloads
 
    When a client has no live network-status documents, it downloads

Modified: tor/trunk/src/or/or.h
===================================================================
--- tor/trunk/src/or/or.h	2006-10-19 23:04:49 UTC (rev 8760)
+++ tor/trunk/src/or/or.h	2006-10-19 23:04:56 UTC (rev 8761)
@@ -188,8 +188,8 @@
 /** How old can a router get before we (as a server) will no longer
  * consider it live? In seconds. */
 #define ROUTER_MAX_AGE_TO_PUBLISH (60*60*20)
-/** How old do we let a saved descriptor get before removing it? */
-#define OLD_ROUTER_DESC_MAX_AGE (60*60*60)
+/** How old do we let a saved descriptor get before force-removing it? */
+#define OLD_ROUTER_DESC_MAX_AGE (60*60*24*5)
 /** How old do we let a networkstatus get before ignoring it? */
 #define NETWORKSTATUS_MAX_AGE (60*60*24)
 

Modified: tor/trunk/src/or/routerlist.c
===================================================================
--- tor/trunk/src/or/routerlist.c	2006-10-19 23:04:49 UTC (rev 8760)
+++ tor/trunk/src/or/routerlist.c	2006-10-19 23:04:56 UTC (rev 8761)
@@ -1999,8 +1999,8 @@
 }
 
 /** Deactivate any routers from the routerlist that are more than
- * ROUTER_MAX_AGE seconds old; remove old routers from the list of
- * cached routers if we have too many.
+ * ROUTER_MAX_AGE seconds old and not recommended by any networkstatuses;
+ * remove old routers from the list of cached routers if we have too many.
  */
 void
 routerlist_remove_old_routers(void)
@@ -2012,32 +2012,37 @@
   routerinfo_t *router;
   signed_descriptor_t *sd;
   digestmap_t *retain;
-  or_options_t *options = get_options();
   if (!routerlist || !networkstatus_list)
     return;
 
   retain = digestmap_new();
   cutoff = now - OLD_ROUTER_DESC_MAX_AGE;
-  if (options->DirPort) {
-    SMARTLIST_FOREACH(networkstatus_list, networkstatus_t *, ns,
-      {
-        SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs,
-          if (rs->published_on >= cutoff)
-            digestmap_set(retain, rs->descriptor_digest, (void*)1));
-      });
-  }
+  /* Build a list of all the descriptors that _anybody_ recommends. */
+  SMARTLIST_FOREACH(networkstatus_list, networkstatus_t *, ns,
+    {
+      SMARTLIST_FOREACH(ns->entries, routerstatus_t *, rs,
+        if (rs->published_on >= cutoff)
+          digestmap_set(retain, rs->descriptor_digest, (void*)1));
+    });
 
-  cutoff = now - ROUTER_MAX_AGE;
-  /* Remove too-old members of routerlist->routers. */
-  for (i = 0; i < smartlist_len(routerlist->routers); ++i) {
-    router = smartlist_get(routerlist->routers, i);
-    if (router->cache_info.published_on <= cutoff &&
-        !digestmap_get(retain, router->cache_info.signed_descriptor_digest)) {
-      /* Too old.  Remove it. */
-      log_info(LD_DIR,
-               "Forgetting obsolete (too old) routerinfo for router '%s'",
-               router->nickname);
-      routerlist_remove(routerlist, router, i--, 1);
+  /* If we have a bunch of networkstatuses, we should consider pruning current
+   * routers that are too old and that nobody recommends.  (If we don't have
+   * enough networkstatuses, then we should get more before we decide to kill
+   * routers.) */
+  if (smartlist_len(networkstatus_list) > get_n_v2_authorities() / 2) {
+    cutoff = now - ROUTER_MAX_AGE;
+    /* Remove too-old unrecommended members of routerlist->routers. */
+    for (i = 0; i < smartlist_len(routerlist->routers); ++i) {
+      router = smartlist_get(routerlist->routers, i);
+      if (router->cache_info.published_on <= cutoff &&
+          !digestmap_get(retain,router->cache_info.signed_descriptor_digest)) {
+        /* Too old: remove it.  (If we're a cache, just move it into
+         * old_routers.) */
+        log_info(LD_DIR,
+                 "Forgetting obsolete (too old) routerinfo for router '%s'",
+                 router->nickname);
+        routerlist_remove(routerlist, router, i--, 1);
+      }
     }
   }
 
@@ -2052,8 +2057,8 @@
     }
   }
 
-  /* Now we're looking at routerlist->old_routers for extraneous
-   * members. (We'd keep all the members if we could, but we'd like to save
+  /* Now we might have to look at routerlist->old_routers for extraneous
+   * members. (We'd keep all the members if we could, but we need to save
    * space.) First, check whether we have too many router descriptors, total.
    * We're okay with having too many for some given router, so long as the
    * total number doesn't approach max_descriptors_per_router()*len(router).
@@ -3595,10 +3600,6 @@
      * But, if we want to have a complete list, fetch it anyway. */
     return 0;
   }
-  if (rs->published_on + ROUTER_MAX_AGE < now) {
-    /* This one is too old to consider. */
-    return 0;
-  }
   if (rs->published_on + ESTIMATED_PROPAGATION_TIME > now) {
     /* Most caches probably don't have this descriptor yet. */
     return 0;