[minion-cvs] Flush hashlogs to disk more often, fix broken DELKEYS, ...

[nickm@seul.org (Nick Mathewson)]

Update of /home/minion/cvsroot/src/minion/lib/mixminion/server
In directory moria.mit.edu:/tmp/cvs-serv21634/src/minion/lib/mixminion/server

Modified Files:
	HashLog.py ServerKeys.py ServerMain.py 
Log Message:
Flush hashlogs to disk more often, fix broken DELKEYS, wait for confused children

Index: HashLog.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/server/HashLog.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- HashLog.py	26 Mar 2003 16:34:36 -0000	1.7
+++ HashLog.py	13 Apr 2003 15:54:42 -0000	1.8
@@ -20,6 +20,9 @@
 # FFFF Two-copy journaling to protect against catastrophic failure that
 # FFFF underlying DB code can't handle.
 
+# We flush the log every MAX_JOURNAL hashes.
+MAX_JOURNAL = 128
+
 # flags to pass to os.open when opening the journal file.
 _JOURNAL_OPEN_FLAGS = os.O_WRONLY|os.O_CREAT|getattr(os,'O_SYNC',0)
 class HashLog:
@@ -107,9 +110,11 @@
         LOG.trace("Logging hash %s", binascii.b2a_hex(hash))
         try:
             self.__lock.acquire()
-            # XXX max journal size?
             self.journal[hash] = 1
             os.write(self.journalFile, hash)
+            #XXXX Make this configurable.
+            if len(self.journal) > MAX_JOURNAL:
+                self.sync()
         finally:
             self.__lock.release()
 

Index: ServerKeys.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/server/ServerKeys.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- ServerKeys.py	28 Mar 2003 15:36:23 -0000	1.15
+++ ServerKeys.py	13 Apr 2003 15:54:42 -0000	1.16
@@ -238,14 +238,18 @@
             expiryStr = ""
 
         cutoff = now - self.keySloppiness
-        dirs = [ os.path.join(self.keyDir,"key_"+name)
-                  for va, vu, name in self.keyIntervals if vu < cutoff ]
 
-        for dirname, (va, vu, name) in zip(dirs, self.keyIntervals):
+        for va, vu, name in self.keyIntervals:
+            if vu >= cutoff:
+                continue
             LOG.info("Removing%s key %s (valid from %s through %s)",
-                        expiryStr, name, formatDate(va), formatDate(vu-3600))
+                     expiryStr, name, formatDate(va), formatDate(vu-3600))
+            dirname = os.path.join(self.keyDir, "key_"+name)
             files = [ os.path.join(dirname,f)
-                                 for f in os.listdir(dirname) ]
+                      for f in os.listdir(dirname) ]
+            hashFiles = [ os.path.join(self.hashDir, "hash_"+name) ,
+                          os.path.join(self.hashDir, "hash_"+name+"jrnl") ]
+            files += [ f for f in hashFiles if os.path.exists(f) ]
             secureDelete(files, blocking=1)
             os.rmdir(dirname)
 

Index: ServerMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/server/ServerMain.py,v
retrieving revision 1.47
retrieving revision 1.48
diff -u -d -r1.47 -r1.48
--- ServerMain.py	28 Mar 2003 15:36:23 -0000	1.47
+++ ServerMain.py	13 Apr 2003 15:54:42 -0000	1.48
@@ -523,6 +523,7 @@
         now = time.time()
 
         scheduledEvents.append( (now + 600, "SHRED") )#FFFF make configurable
+        scheduledEvents.append( (now + 180, "WAIT") )#FFFF make configurable
         scheduledEvents.append( (self.mmtpServer.getNextTimeoutTime(now),
                                  "TIMEOUT") )
         nextMix = self.mixPool.getNextMixTime(now)
@@ -579,6 +580,12 @@
             elif event == 'SHRED':
                 self.cleanQueues()
                 insort(scheduledEvents, (now + 600, "SHRED"))
+            elif event == 'WAIT':
+                # Every few minutes, we reap zombies.  Why, you ask?  Isn't
+                # catching sigchild enough?  Nope -- sometimes buggy delivery
+                # software forks, 
+                waitForChildren(blocking=0)
+                insort(scheduledEvents, (now + 180, "WAIT"))
             elif event == 'MIX':
                 # Before we mix, we need to log the hashes to avoid replays.
                 try:
@@ -605,7 +612,7 @@
                 insort(scheduledEvents, (nextMix, "MIX"))
                 LOG.trace("Next mix at %s", formatTime(nextMix,1))
             else:
-                assert event in ("MIX", "SHRED", "TIMEOUT")
+                assert event in ("MIX", "SHRED", "TIMEOUT", "WAIT")
 
     def cleanQueues(self):
         """Remove all deleted messages from queues"""