[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[minion-cvs] The server now runs for me. (Don"t worry -- it will ru...



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv8199/lib/mixminion

Modified Files:
	Common.py Config.py Crypto.py MMTPServer.py Main.py Modules.py 
	Queue.py ServerInfo.py ServerMain.py test.py testSupport.py 
Log Message:
The server now runs for me.  (Don't worry -- it will run for you next.)

I still need to write the client CLI, and test actual delivery before I
can more on to the testing/cleanup phase of 0.0.1.

ServerMain.py:
	- Document Keyring and delivery queues
	- Add keygen functionality
	- Add keygen CLI
	- Make key directories if they don't exist
	- Generate identity keys as needed
	- Check for overlap/gaps in key lifetimes
	- Bugfix: Make getLiveKey work
	- Resolve typos; clarify names
	- Generate keys on startup (temporary measure)

Makefile:
	fix openssl url

Common.py:
	-Bugfix on log
	-Factor out some time-handling functions (mkgmtime, previousMidnight)

Config.py:
	- Minor tweaks.
	- Add 'IdentityKeyBits' configuration option
	
Crypto.py:
	- Add comments to RNG functions

MMTPServer.py:
	- Bugfix: take TLS context as argument, not from config
	- Bugfix: Listen on selected IP, not 0.0.0.0

Main.py:
	- Add server-keygen command

Modules.py:
	- Bugfix: importing from nested package

Queue.py:
	- "Cottrell" mixing wasn't really cottrell mixing.  Now it is.

ServerInfo.py:
	- Spec conformance: key lifetimes must begin at midnight GMT

test.py:
	- Factor our supsendLog/resumeLog.
	- Add tests for some common  stuff
	- Add tests for keyrings

testSupport.py:
	- Debug.


Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Common.py	25 Aug 2002 06:10:33 -0000	1.18
+++ Common.py	29 Aug 2002 03:30:21 -0000	1.19
@@ -307,7 +307,10 @@
     def _log(self, severity, message, args):
         if _SEVERITIES.get(severity, 100) < self.severity:
             return
-        m = message % args
+	if args is None:
+	    m = message
+	else:
+	    m = message % args
         for h in self.handlers:
             h.write(severity, m)
 
@@ -337,7 +340,7 @@
 	indented = "".join(formatted)
 	if indented.endswith('\n'):
 	    indented = indented[:-1]
-	self._log(severity, indented, ())
+	self._log(severity, indented, None)
 
     def error_exc(self, (exclass, ex, tb), message=None, *args):
 	self.log_exc("ERROR", (exclass, ex, tb), message, *args)
@@ -353,6 +356,23 @@
         _THE_LOG = Log('WARN')
 
     return _THE_LOG
+
+#----------------------------------------------------------------------
+# Time processing
+
+def mkgmtime(yyyy,MM,dd,hh,mm,ss):
+    """Analogously to time.mktime, return a number of seconds since the
+       epoch when GMT is yyyy/MM/dd hh:mm:ss"""
+    
+    # we set the DST flag to zero so that subtracting time.timezone always
+    # gives us gmt.
+    return time.mktime((yyyy,MM,dd,hh,mm,ss,0,0,0))-time.timezone
+
+def previousMidnight(when):
+    """Given a time_t 'when', return the greatest time_t <= when that falls
+       on midnight, GMT."""
+    yyyy,MM,dd = time.gmtime(when)[0:3]
+    return mkgmtime(yyyy,MM,dd,0,0,0)
 
 #----------------------------------------------------------------------
 # Signal handling

Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- Config.py	25 Aug 2002 05:58:02 -0000	1.12
+++ Config.py	29 Aug 2002 03:30:21 -0000	1.13
@@ -124,7 +124,7 @@
 def _parseInt(integer):
     """Validation function.  Converts a config value to an int.
        Raises ConfigError on failure."""
-    i = integer.strip().lower()
+    i = integer.strip()
     try:
         return int(i)
     except ValueError, _:
@@ -268,9 +268,7 @@
 	    (0 <= mm < 60)  and (0 <= ss <= 61)):
 	raise ConfigError("Invalid %s %r" % (("date","time")[_timeMode],s))
 
-    # we set the DST flag to zero so that subtracting time.timezone always
-    # gives us gmt.
-    return time.mktime((yyyy,MM,dd,hh,mm,ss,0,0,0))-time.timezone
+    return mixminion.Common.mkgmtime(yyyy, MM, dd, hh, mm, ss)
 
 def _parseTime(s):
     """Validation function.  Converts from DD/MM/YYYY HH:MM:SS format
@@ -644,7 +642,8 @@
                      'LogLevel' : ('ALLOW', _parseSeverity, "WARN"),
                      'EchoMessages' : ('ALLOW', _parseBoolean, "no"),
                      'EncryptIdentityKey' : ('REQUIRE', _parseBoolean, "yes"),
-                     'PublicKeyLifetime' : ('REQUIRE', _parseInterval,
+		     'IdentityKeyBits': ('ALLOW', _parseInt, "2048"),
+                     'PublicKeyLifetime' : ('ALLOW', _parseInterval,
                                             "30 days"),
                      'PublicKeySloppiness': ('ALLOW', _parseInterval,
                                              "5 minutes"),

Index: Crypto.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Crypto.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Crypto.py	25 Aug 2002 06:10:34 -0000	1.18
+++ Crypto.py	29 Aug 2002 03:30:21 -0000	1.19
@@ -427,6 +427,8 @@
         # FFFF This implementation is about 2-4x as good as the last one, but
 	# FFFF still could be better.  It's faster than getFloat()*max.
 
+	# XXXX (This code assumes that integers are at least 32 bits.)
+
         assert 0 < max < 0x3fffffff
 	_ord = ord
 	while 1:
@@ -440,12 +442,12 @@
 		return o % max
 
     def getFloat(self):
-	"""Return a floating-point number between 0 and 1.  The number
-	   will have 'bytes' bytes of resolution."""
+	"""Return a floating-point number between 0 and 1."""
 	b = self.getBytes(4)
 	_ord = ord
 	o = ((((((_ord(b[0])&0x7f)<<8) + _ord(b[1]))<<8) + 
 	      _ord(b[2]))<<8) + _ord(b[3])
+	#return o / float(0x7fffffff)
 	return o / 2147483647.0
 
     def _prng(self, n):

Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- MMTPServer.py	25 Aug 2002 05:58:02 -0000	1.14
+++ MMTPServer.py	29 Aug 2002 03:30:21 -0000	1.15
@@ -620,14 +620,14 @@
 class MMTPServer(AsyncServer):
     """A helper class to invoke AsyncServer, MMTPServerConnection, and
        MMTPClientConnection"""
-    def __init__(self, config):
+    def __init__(self, config, tls):
 	AsyncServer.__init__(self)
 
-        self.context = config.getTLSContext(server=1)
+        self.context = tls
 	# FFFF Don't always listen; don't always retransmit!
 	# FFFF Support listening on specific IPs
-        self.listener = ListenConnection("0.0.0.0",
-                                         config['Outgoing/MMTP']['Port'],
+        self.listener = ListenConnection(config['Incoming/MMTP']['IP'],
+                                         config['Incoming/MMTP']['Port'],
 					 LISTEN_BACKLOG,
                                          self._newMMTPConnection)
 	#self.config = config

Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- Main.py	25 Aug 2002 05:58:02 -0000	1.2
+++ Main.py	29 Aug 2002 03:30:21 -0000	1.3
@@ -75,7 +75,8 @@
 _COMMANDS = {
     "unittests" : ( 'mixminion.test', 'testAll' ),
     "benchmarks" : ( 'mixminion.benchmark', 'timeAll' ),
-    "server" : ( 'mixminion.ServerMain', 'runServer' )
+    "server" : ( 'mixminion.ServerMain', 'runServer' ),
+    "server-keygen" : ( 'mixminion.ServerMain', 'runKeygen')
 }
 
 def main(args):

Index: Modules.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Modules.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- Modules.py	25 Aug 2002 05:58:02 -0000	1.9
+++ Modules.py	29 Aug 2002 03:30:21 -0000	1.10
@@ -218,7 +218,7 @@
         try:
 	    sys.path[0:0] = self.path
 	    try:
-		m = __import__(pyPkg, {}, {}, [])
+		m = __import__(pyPkg, {}, {}, [pyClassName])
 	    except ImportError, e:
 		raise MixError("%s while importing %s" %(str(e),className))
         finally:

Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- Queue.py	25 Aug 2002 05:58:02 -0000	1.15
+++ Queue.py	29 Aug 2002 03:30:21 -0000	1.16
@@ -373,19 +373,19 @@
     """A CottrellMixQueue holds a group of files, and returns some of them
        as requested, according the Cottrell (timed dynamic-pool) mixing
        algorithm from Mixmaster."""
-    def __init__(self, location, interval=600, minPoolSize=6, maxSendRate=.3):
+    def __init__(self, location, interval=600, threshold=6, retainRate=.7):
 	"""Create a new queue that yields a batch of message every 'interval'
-	   seconds, never allows its pool size to drop below 'minPoolSize',
-	   and never sends more than maxSendRate * the current pool size."""
+	   seconds, never sends unless it has more than <threshold> messages,
+	   and always keeps <retainRate> * the current pool size."""
 	TimedMixQueue.__init__(self, location, interval)
-	self.minPoolSize = minPoolSize
-	self.maxBatchSize = int(maxSendRate*minPoolSize)
-	if self.maxBatchSize < 1: 
-	    self.maxBatchSize = 1
+	self.threshold = threshold
+	self.sendRate = 1.0 - retainRate
 
     def getBatch(self):
 	pool = self.count()
-	nTransmit = min(pool-self.minPoolSize, self.maxBatchSize)
+	if pool <= self.threshold:
+	    return []
+	nTransmit = int(pool * self.sendRate)
 	return self.pickRandom(nTransmit)
 
 class BinomialCottrellMixQueue(CottrellMixQueue):
@@ -393,8 +393,9 @@
        from the pool of size P, sends each message with probability N/P."""
     def getBatch(self):
 	pool = self.count()
-	nTransmit = min(pool-self.minPoolSize, self.maxBatchSize)
-	msgProbability = float(nTransmit) / pool
+	if pool <= self.threshold:
+	    return []
+	msgProbability = self.sendRate
 	return self.rng.shuffle([ h for h in self.getAllMessages() 
 				    if self.rng.getFloat() < msgProbability ])
 

Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- ServerInfo.py	25 Aug 2002 06:10:35 -0000	1.12
+++ ServerInfo.py	29 Aug 2002 03:30:21 -0000	1.13
@@ -250,6 +250,9 @@
     if not validAt:
         validAt = time.time()
 
+    # Round validAt to previous mignight.
+    validAt = mixminion.Common.previousMidnight(validAt+30)
+
     validUntil = validAt + config['Server']['PublicKeyLifetime'][2]
     certStarts = validAt - CERTIFICATE_EXPIRY_SLOPPINESS
     certEnds = validUntil + CERTIFICATE_EXPIRY_SLOPPINESS + \

Index: ServerMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerMain.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- ServerMain.py	25 Aug 2002 05:58:02 -0000	1.6
+++ ServerMain.py	29 Aug 2002 03:30:21 -0000	1.7
@@ -14,11 +14,13 @@
 import bisect
 
 import mixminion._minionlib
+import mixminion.Crypto
 import mixminion.Queue
 import mixminion.MMTPServer
-from mixminion.ServerInfo import ServerKeyset, ServerInfo
+from mixminion.ServerInfo import ServerKeyset, ServerInfo, _date, \
+     generateServerDescriptorAndKeys
 from mixminion.Common import getLog, MixFatalError, MixError, secureDelete, \
-     createPrivateDir
+     createPrivateDir, previousMidnight, ceilDiv
 
 # Directory layout:
 #     MINION_HOME/work/queues/incoming/
@@ -28,7 +30,8 @@
 #                      tls/dhparam
 #                      hashlogs/hash_1 ...
 #                 log
-#                 keys/key_1/ServerDesc
+#                 keys/identity.key
+#                      key_1/ServerDesc
 #                            mix.key
 #                            mmtp.key
 #                            mmtp.cert
@@ -37,28 +40,60 @@
 #                       ....
 
 class ServerKeyring:
-    # homeDir: ----
-    # keyDir: ----
-    # keySloppiness: ----
-    # keyIntervals: list of (start, end, ServerKeyset Name)
+    """A ServerKeyRing remembers current and future keys, descriptors, and 
+       hash logs for a mixminion server.
+       
+       FFFF: We need a way to generate keys as needed
+       """
+    # homeDir: server home directory 
+    # keyDir: server key directory
+    # keySloppiness: fudge-factor: how forgiving are we about key liveness?
+    # keyIntervals: list of (start, end, keyset Name)
+    # liveKey: list of (start, end, keyset name for current key.)
+    # nextRotation: time_t when this key expires.
+    # keyRange: tuple of (firstKey, lastKey) to represent which key names
+    #      have keys on disk.
+
     def __init__(self, config):
+	"Create a ServerKeyring from a config object"
 	self.configure(config)
 
     def configure(self, config):
+	"Set up a SeverKeyring from a config object"
+	self.config = config
 	self.homeDir = config['Server']['Homedir']
 	self.keyDir = os.path.join(self.homeDir, 'keys')
-	self.keySloppiness = config['Server']['PublicKeySloppiness']
+	self.hashDir = os.path.join(self.homeDir, 'work', 'hashlogs')
+	self.keySloppiness = config['Server']['PublicKeySloppiness'][2]
 	self.checkKeys()
 
     def checkKeys(self):
+	"""Internal method: read information about all this server's
+	   currently-prepared keys from disk."""
         self.keyIntervals = [] 
+	firstKey = sys.maxint
+	lastKey = 0
+
+	if not os.path.exists(self.keyDir):
+	    createPrivateDir(self.keyDir)
+
         for dirname in os.listdir(self.keyDir):
+	    if not os.path.isdir(os.path.join(self.keyDir,dirname)):
+		continue
             if not dirname.startswith('key_'):
 		getLog().warn("Unexpected directory %s under %s",
 			      dirname, self.keyDir)
                 continue
             keysetname = dirname[4:]
-            
+	    try:
+		setNum = int(keysetname)
+		if setNum < firstKey: firstKey = setNum
+		if setNum > lastKey: lastKey = setNum
+	    except ValueError, _:
+		getLog().warn("Unexpected directory %s under %s",
+			      dirname, self.keyDir)
+		continue
+
             d = os.path.join(self.keyDir, dirname)
             si = os.path.join(d, "ServerDesc")
             if os.path.exists(si):
@@ -66,10 +101,94 @@
                 t1 = inf['Server']['Valid-After']
                 t2 = inf['Server']['Valid-Until']
                 self.keyIntervals.append( (t1, t2, keysetname) ) 
+	    else:
+		getLog().warn("No server descriptor found for key %s"%dirname)
 
         self.keyIntervals.sort()
-    
+	self.keyRange = (firstKey, lastKey)
+
+	# Now we try to see whether we have more or less than 1 key in effect
+	# for a given time.
+	for idx in xrange(len(self.keyIntervals)-1):
+	    end = self.keyIntervals[idx][1]
+	    start = self.keyIntervals[idx+1][0]
+	    if start < end:
+		getLog().warn("Multiple keys for %s.  That's unsupported.",
+			      _date(end))
+	    elif start > end:
+		getLog().warn("Gap in key schedule: no key from %s to %s",
+			      _date(end), _date(start))
+
+	self.nextKeyRotation = 0 # Make sure that now > nextKeyRotation before
+	                         # we call _getLiveKey()
+	self._getLiveKey()       # Set up liveKey, nextKeyRotation.
+
+    def getIdentityKey(self):
+	"""Return this server's identity key.  Generate one if it doesn't 
+	   exist."""
+	password = None # FFFF
+	fn = os.path.join(self.keyDir, "identity.key")
+	bits = self.config['Server']['IdentityKeyBits']
+	if os.path.exists(fn):
+	    key = mixminion.Crypto.pk_PEM_load(fn, password)
+	    keylen = key.get_modulus_bytes()*8
+	    if keylen != bits:
+		getLog().warn(
+		    "Stored identity key has %s bits, but you asked for %s.",
+		    keylen, bits)
+	else:
+	    getLog().info("Generating identity key. (This may take a while.)")
+	    key = mixminion.Crypto.pk_generate(bits)
+	    mixminion.Crypto.pk_PEM_save(key, fn, password)
+	    getLog().info("Generated %s-bit identity key.", bits)
+	
+	return key
+
+    def createKeys(self, num=1, startAt=None):
+	"""Generate 'num' public keys for this server. If startAt is provided,
+           make the first key become valid at'startAt'.  Otherwise, make the
+	   first key become valid right after the last key we currently have
+	   expires.  If we have no keys now, make the first key start now."""
+	password = None #FFFF
+
+	if startAt is None:
+	    if self.keyIntervals:
+		startAt = self.keyIntervals[-1][1]+60
+	    else:
+		startAt = time.time()+60
+
+	startAt = previousMidnight(startAt)
+	
+	firstKey, lastKey = self.keyRange
+
+	for i in xrange(num):
+	    if firstKey == sys.maxint:
+		keynum = firstkey = lastkey = 1
+	    elif firstKey > 1:
+		firstKey -= 1
+		keynum = firstKey
+	    else:
+		lastKey += 1
+		keynum = lastKey
+
+	    keyname = "%04d" % keynum
+
+	    nextStart = startAt + self.config['Server']['PublicKeyLifetime'][2]
+
+	    getLog().info("Generating key %s to run from %s through %s", 
+			  keyname, _date(startAt), _date(nextStart-3600))
+ 	    generateServerDescriptorAndKeys(config=self.config,
+					    identityKey=self.getIdentityKey(),
+					    keyname=keyname,
+					    keydir=self.keyDir,
+					    hashdir=self.hashDir,
+					    validAt=startAt)
+	    startAt = nextStart
+
+        self.checkKeys()
+
     def removeDeadKeys(self):
+	"""Remove all keys that have expired"""
         now = time.time()
         cutoff = now - self.keySloppiness
 	dirs = [ os.path.join(self.keyDir,"key_"+name)
@@ -83,24 +202,44 @@
 	    
 	self.checkKeys()
 
-    def _getLiveKey(self):
-	# returns valid-after, valid-until, name
-        now = time.time()
-        idx = bisect.bisect_left(self.keyIntervals, (now, None, None))
-        return self.keyIntervals[idx]
+    def _getLiveKey(self, when=None):
+	"""Find the first key that is now valid.  Return (Valid-after,
+	   valid-util, name)."""
+        if not self.keyIntervals:
+	    self.liveKey = None
+	    self.nextKeyRotation = 0
+	    return None
+
+
+	w = when
+	if when is None: 
+	    when = time.time()
+	    if when < self.nextKeyRotation:
+		return self.liveKey
+
+	idx = bisect.bisect_right(self.keyIntervals, (when, None, None))-1
+	k = self.keyIntervals[idx]
+	if w is None:
+	    self.liveKey = k
+	    self.nextKeyRotation = k[1]
+		
+	return k
 
     def getNextKeyRotation(self):
-        return self._getLiveKey()[1]
+	"""Return the expiration time of the current key"""
+        return self.nextKeyRotation
 
     def getServerKeyset(self):
+	"""Return a ServerKeyset object for  the currently live key."""
 	# FFFF Support passwords on keys
 	_, _, name = self._getLiveKey()
-	hashroot = os.path.join(self.homeDir, 'work', 'hashlogs')
-	keyset = ServerKeyset(self.keyDir, name, hashroot)
+	keyset = ServerKeyset(self.keyDir, name, self.hashDir)
 	keyset.load()
-	return self.keyset
+	return keyset
 	
     def getDHFile(self):
+	"""Return the filename for the diffie-helman parameters for the
+	   server.  Creates the file if it doesn't yet exist."""
 	dhdir = os.path.join(self.homeDir, 'work', 'tls')
 	createPrivateDir(dhdir)
 	dhfile = os.path.join(dhdir, 'dhparam')
@@ -112,37 +251,48 @@
         return dhfile
 			    
     def getTLSContext(self):
+	"""Create and return a TLS context from the currently live key."""
         keys = self.getServerKeyset()
         return mixminion._minionlib.TLSContext_new(keys.getCertFileName(),
-						   keys.GetMMTPKey(),
+						   keys.getMMTPKey(),
 						   self.getDHFile())
 
     def getPacketHandler(self):
+	"""Create and return a PacketHandler from the currently live key."""
         keys = self.getServerKeyset()
         return mixminion.PacketHandler.PacketHandler(keys.getPacketKey(),
-                                                     keys.getHashLogFile())
+                                                     keys.getHashLogFileName())
 
 class IncomingQueue(mixminion.Queue.DeliveryQueue):
+    """A DeliveryQueue to accept messages from incoming MMTP connections,
+       process them with a packet handler, and send them into a mix pool."""
+
     def __init__(self, location, packetHandler):
+	"""Create an IncomingQueue that stores its messages in <location>
+	   and processes them through <packetHandler>."""
 	mixminion.Queue.DeliveryQueue.__init__(self, location)
 	self.packetHandler = packetHandler
-	self.mixQueue = None
+	self.mixPool = None
 
-    def connectQueues(self, mixQueue):
-	self.mixQueue = mixQueue
+    def connectQueues(self, mixPool):
+	"""Sets the target mix queue"""
+	self.mixPool = mixPool
 
     def queueMessage(self, msg):
-	mixminion.Queue.queueMessage(None, msg)
+	"""Add a message for delivery"""
+	mixminion.DeliveryQueue.queueMessage(None, msg)
     
     def deliverMessages(self, msgList):
+	"Implementation of abstract method from DeliveryQueue."
 	ph = self.packetHandler
 	for handle, _, message, n_retries in msgList:
 	    try:
 		res = ph.packetHandler(message)
 		if res is None:
+		    # Drop padding before it gets to the mix.
 		    getLog().info("Padding message dropped")
 		else:
-		    self.mixQueue.queueObject(res)
+		    self.mixPool.queueObject(res)
 		    self.deliverySucceeded(handle)
 	    except mixminion.Crypto.CryptoError, e:
 		getLog().warn("Invalid PK or misencrypted packet header:"+str(e))
@@ -154,17 +304,28 @@
 		getLog().warn("Discarding bad packet:"+str(e))
 		self.deliveryFailed(handle)
 
-class MixQueue:
+class MixPool:
+    """Wraps a mixminion.Queue.*MixQueue to send messages to an exit queue
+       and a delivery queue."""
     def __init__(self, queue):
+	"""Create a new MixPool to wrap a given *MixQueue."""
 	self.queue = queue
 	self.outgoingQueue = None
 	self.moduleManager = None
 
+    def queueObject(self, obj):
+	"""Insert an object into the queue."""
+	self.queue.queueObject(ob)
+
     def connectQueues(self, outgoing, manager):
+	"""Sets the queue for outgoing mixminion packets, and the
+  	   module manager for deliverable messages."""
 	self.outgoingQueue = outgoing
 	self.moduleManager = manager
 
     def mix(self):
+	"""Get a batch of messages, and queue them for delivery as 
+	   appropriate."""
 	handles = self.queue.getBatch()
 	for h in handles:
 	    tp, info = self.queue.getObject(h)
@@ -177,14 +338,20 @@
 		self.outgoingQueue.queueMessage(ipv4, msg)
 
 class OutgoingQueue(mixminion.Queue.DeliveryQueue):
+    """DeliveryQueue to send messages via outgoing MMTP connections."""
     def __init__(self, location):
-	OutgoingQueue.__init__(self, location)
+	"""Create a new OutgoingQueue that stores its messages in a given
+ 	   location."""
+        mixminion.Queue.DeliveryQueue.__init__(self, location)
 	self.server = None
 
     def connectQueues(self, server):
+	"""Set the MMTPServer that this OutgoingQueue informs of its 
+	   deliverable messages."""
 	self.server = server
 
     def deliverMessages(self, msgList):
+	"Implementation of abstract method from DeliveryQueue."
 	# Map from addr -> [ (handle, msg) ... ]
 	msgs = {}
 	for handle, addr, message, n_retries in msgList:
@@ -194,9 +361,11 @@
 	    self.server.sendMessages(addr.ip, addr.port, addr.keyinfo,
 				     messages, handles)
 
-class _MMTPConnection(mixminion.MMTPServer):
-    def __init__(self, config):
-        MMTPServer.__init__(self, config)
+class _MMTPServer(mixminion.MMTPServer.MMTPServer):
+    """Implementation of mixminion.MMTPServer that knows about
+       delivery queues."""
+    def __init__(self, config, tls):
+        mixminion.MMTPServer.MMTPServer.__init__(self, config, tls)
 
     def connectQueues(self, incoming, outgoing):
         self.incomingQueue = incoming
@@ -211,14 +380,24 @@
     def onMessageUndeliverable(self, msg, handle, retriable):
 	self.outgoingQueue.deliveryFailed(handle, retriable)
 
-
 class MixminionServer:
+    """Wraps and drives all the queues, and the async net server.  Handles
+       all timed events."""
     def __init__(self, config):
+	"""Create a new server from a ServerConfig."""
 	self.config = config
 	self.keyring = ServerKeyring(config)
-	
+	if self.keyring._getLiveKey() is None:
+	    getLog().info("Generating a month's worth of keys.")
+	    getLog().info("(Don't count on this feature in future versions.)")
+	    # We might not be able to do this, if we password-encrypt keys
+	    keylife = config['Server']['PublicKeyLifetime'][2]
+	    nKeys = ceilDiv(30*24*60*60, keylife)
+	    self.keyring.createKeys(nKeys)
+	    
 	self.packetHandler = self.keyring.getPacketHandler()
-	self.mmtpConnection = _MMTPConnection(config)
+	tlsContext = self.keyring.getTLSContext()
+	self.mmtpServer = _MMTPServer(config, tlsContext)
 
 	# FFFF Modulemanager should know about async so it can patch in if it
 	# FFFF needs to.
@@ -232,29 +411,32 @@
 
 	mixDir = os.path.join(queueDir, "mix")
 	# FFFF The choice of mix algorithm should be configurable
-	self.mixQueue = MixQueue(mixminion.Queue.TimedMixQueue(mixDir, 60))
+	self.mixPool = MixPool(mixminion.Queue.TimedMixQueue(mixDir, 60))
 
 	outgoingDir = os.path.join(queueDir, "outgoing")
 	self.outgoingQueue = OutgoingQueue(outgoingDir)
 
-	self.incomingQueue.connectQueues(mixQueue=self.mixQueue)
-	self.mixQueue.connectQueues(outgoing=self.outgoingQueue,
-				    manager=self.moduleManager)
-	self.outgoingQueue.connectQueues(server=self.mmtpConnection)
-	self.mmtpConnection.connectQueues(incoming=self.incomingQueue,
-					  outgoing=self.outgoingQueue)
+	self.incomingQueue.connectQueues(mixPool=self.mixPool)
+	self.mixPool.connectQueues(outgoing=self.outgoingQueue,
+				   manager=self.moduleManager)
+	self.outgoingQueue.connectQueues(server=self.mmtpServer)
+	self.mmtpServer.connectQueues(incoming=self.incomingQueue,
+				      outgoing=self.outgoingQueue)
 	
     def run(self):
+	"""Run the server; don't return unless we hit an exception."""
+	# FFFF Use heapq to schedule events?
 	now = time.time()
 	nextMix = now + 60 # FFFF Configurable!
 	nextShred = now + 6000
-	
+	nextRotate = self.keyring.getNextKeyRotation() # FFFF use this.
 	while 1:
 	    while time.time() < nextMix:
-		self.mmtpConnection.process(1)
+		self.mmtpServer.process(1)
 		self.incomingQueue.sendReadyMessages()
 	    
-	    self.mixQueue.mix()
+	    getLog().trace("Mix interval elapsed")
+	    self.mixPool.mix()
 	    self.outgoingQueue.sendReadyMessages()
 	    self.moduleManager.sendReadyMessages()
 
@@ -263,7 +445,7 @@
 	    if now > nextShred:
 		# Configurable shred interval
 		self.incomingQueue.cleanQueue()
-		self.mixQueue.queue.cleanQueue()
+		self.mixPool.queue.cleanQueue()
 		self.outgoingQueue.cleanQueue()
 		self.moduleManager.cleanQueues()
 		nextShred = now + 6000
@@ -276,7 +458,7 @@
     print >>sys.stderr, "Usage: %s [-h] [-f configfile]" % cmd
     sys.exit(0)
 
-def configFromArgs(cmd, args):
+def configFromServerArgs(cmd, args):
     options, args = getopt.getopt(args, "hf:", ["help", "config="])
     if args:
 	usageAndExit(cmd)
@@ -286,20 +468,23 @@
 	    usageAndExit()
 	if o in ('-f', '--config'):
 	    configFile = v
+
+    return readConfigFile(configFile)
+
+def readConfigFile(configFile):
     try:
-	config = mixminion.Config.ServerConfig(fname=configFile)
+	return mixminion.Config.ServerConfig(fname=configFile)
     except (IOError, OSError), e:
-	print >>sys.stderr, "Error reading configuration file %r"%configFile
+	print >>sys.stderr, "Error reading configuration file %r:"%configFile
+	print >>sys.stderr, "   ", str(e)
 	sys.exit(1)
     except mixminion.Config.ConfigError, e:
 	print >>sys.stderr, "Error in configuration file %r"%configFile
 	print >>sys.stderr, str(e)
 	sys.exit(1)
 
-    return config
-
 def runServer(cmd, args):
-    config = configFromArgs(cmd, args)
+    config = configFromServerArgs(cmd, args)
     try:
 	mixminion.Common.getLog().configure(config)
 	getLog().debug("Configuring server")
@@ -309,7 +494,7 @@
 
 	server = MixminionServer(config)
     except:
-	getLog().fatal_exc("Exception while configuring server")
+	getLog().fatal_exc(sys.exc_info(),"Exception while configuring server")
 	print >>sys.stderr, "Shutting down because of exception"
 	sys.exit(1)
 
@@ -319,7 +504,40 @@
     except KeyboardInterrupt:
 	pass
     except:
-	getLog().fatal_exc("Exception while running server")
+	getLog().fatal_exc(sys.exc_info(),"Exception while running server")
     getLog().info("Server shutting down")
     
     sys.exit(0)
+
+#----------------------------------------------------------------------
+def runKeygen(cmd, args):
+    options, args = getopt.getopt(args, "hf:n:", ["help", "config=", "keys="])
+    # FFFF password-encrypted keys
+    keys=1
+    usage=0
+    configFile = '/etc/miniond.conf'
+    for opt,val in options:
+	if opt in ('-h', '--help'):
+	    usage=1
+	elif opt in ('-f', '--config'):
+	    configFile = val
+	elif opt in ('-n', '--keys'):
+	    try:
+		keys = int(val)
+	    except ValueError, _:
+		print >>sys.stderr,("%s requires an integer" %opt)
+		sys.exit(1)
+	if usage:
+	    print >>sys.stderr, "Usage: %s [-h] [-f configfile] [-n nKeys]"%cmd
+	    sys.exit(1)
+	config = readConfigFile(configFile)
+
+    getLog().setMinSeverity("INFO")
+    mixminion.Crypto.init_crypto(config)
+    keyring = ServerKeyring(config)
+    print >>sys.stderr, "Creating %s keys..." % keys
+    for i in xrange(keys):
+	keyring.createKeys(1)
+	print >> sys.stderr, ".... (%s/%s done)" % (i+1,keys)
+    
+    

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- test.py	25 Aug 2002 06:10:35 -0000	1.24
+++ test.py	29 Aug 2002 03:30:21 -0000	1.25
@@ -22,6 +22,7 @@
 import base64
 import stat
 import cPickle
+import cStringIO
 
 from mixminion.testSupport import mix_mktemp
 from mixminion.Common import MixError, MixFatalError, MixProtocolError, getLog
@@ -46,6 +47,73 @@
 def floatEq(f1,f2):
     return abs(f1-f2) < .00001
 
+def suspendLog():
+    """Temporarily suppress logging output."""
+    log = getLog()
+    if hasattr(log, '_storedHandlers'):
+	resumeLog()
+    buf = cStringIO.StringIO()
+    h = mixminion.Common._ConsoleLogHandler(cStringIO.StringIO())
+    log._storedHandlers = log.handlers
+    log._testBuf = buf
+    log.handlers = []
+    log.addHandler(h)
+
+def resumeLog():
+    """Resume logging output.  Return all new log messages since the last
+       suspend."""
+    log = getLog()
+    if not hasattr(log, '_storedHandlers'):
+	return None
+    buf = log._testBuf
+    del log._testBuf
+    log.handlers = log._storedHandlers
+    del log._storedHandlers
+    return str(buf)
+
+#----------------------------------------------------------------------
+# Tests for common functionality
+
+class MiscTests(unittest.TestCase):
+    def testDiv(self):
+	from mixminion.Common import floorDiv, ceilDiv
+
+	self.assertEquals(floorDiv(10,1), 10)
+	self.assertEquals(floorDiv(10,2), 5)
+	self.assertEquals(floorDiv(10,3), 3)
+	self.assertEquals(floorDiv(10,11), 0)
+	self.assertEquals(floorDiv(0,11), 0)
+	self.assertEquals(floorDiv(-1,1), -1)
+	self.assertEquals(floorDiv(-1,2), -1)
+	self.assertEquals(floorDiv(-10,3), -4)
+	self.assertEquals(floorDiv(-10,-3), 3)
+
+	self.assertEquals(ceilDiv(10,1), 10)
+	self.assertEquals(ceilDiv(10,2), 5)
+	self.assertEquals(ceilDiv(10,3), 4)
+	self.assertEquals(ceilDiv(10,11), 1)
+	self.assertEquals(ceilDiv(0,11), 0)
+	self.assertEquals(ceilDiv(-1,1), -1)
+	self.assertEquals(ceilDiv(-1,2), 0)
+	self.assertEquals(ceilDiv(-10,3), -3)
+	self.assertEquals(ceilDiv(-10,-3), 4)
+
+    def testTimeFns(self):
+	from mixminion.Common import floorDiv, mkgmtime, previousMidnight
+	# This isn't a very good test.
+	now = int(time.time())
+	max_sec_per_day = 24*60*60+ 1
+	for t in xrange(10, now, floorDiv(now, 1000)):
+	    yyyy,MM,dd,hh,mm,ss = time.gmtime(t)[:6]
+	    self.assertEquals(t, mkgmtime(yyyy,MM,dd,hh,mm,ss))
+	    pm = previousMidnight(t)
+	    yyyy2,MM2,dd2,hh2,mm2,ss2 = time.gmtime(pm)[:6]
+	    self.assertEquals((yyyy2,MM2,dd2), (yyyy,MM,dd))
+	    self.assertEquals((0,0,0), (hh2,mm2,ss2))
+	    self.failUnless(pm <= t and 0 <= (t-pm) <= max_sec_per_day)
+	    self.assertEquals(previousMidnight(t), pm)
+	    self.assertEquals(previousMidnight(pm), pm)
+
 #----------------------------------------------------------------------
 import mixminion._minionlib as _ml
 
@@ -367,7 +435,7 @@
 
     def test_aesprng(self):
         # Make sure that AESCounterPRNG is really repeatable.
-        key ="aaaa"*4
+        key ="aaab"*4
         PRNG = AESCounterPRNG(key)
         self.assert_(prng(key,100000) == (
                           PRNG.getBytes(5)+PRNG.getBytes(16*1024-5)+
@@ -380,6 +448,15 @@
         for i in xrange(1,10000,17):
             self.failUnless(0 <= PRNG.getInt(10) < 10)
             self.failUnless(0 <= PRNG.getInt(i) < i)
+
+##  	itot=ftot=0
+##  	for i in xrange(1000000):
+##  	    itot += PRNG.getInt(10)
+##  	    ftot += PRNG.getFloat()
+
+##  	print "AVG INT", itot/1000000.0
+##  	print "AVG FLT", ftot/1000000.0
+	
 	for i in xrange(100):
 	    self.failUnless(0 <= PRNG.getFloat() < 1)
 
@@ -1403,8 +1480,8 @@
 	b.sort()
 	self.assertEquals(msgs,b)
 	
-	cmq = CottrellMixQueue(d_m, 600, 6, .5)
-	# Not enough messages
+	cmq = CottrellMixQueue(d_m, 600, 6, .7)
+	# Not enough messages (<= 6)
 	self.assertEquals([], cmq.getBatch())
 	self.assertEquals([], cmq.getBatch())
 	# 8 messages: 2 get sent
@@ -1421,19 +1498,19 @@
 	    if b != b1:
 		allEq = 0; break
 	self.failIf(allEq)
-	# Don't send more than 3.
-	for x in xrange(100):
+	# Send 30 when there are 100 messages.
+	for x in xrange(92):
 	    cmq.queueMessage("Hello2 %s"%x)
 	for x in xrange(10):
-	    self.assertEquals(3, len(cmq.getBatch()))
+	    self.assertEquals(30, len(cmq.getBatch()))
 
-	bcmq = BinomialCottrellMixQueue(d_m, 600, 6, .5)
-	allThree = 1
+	bcmq = BinomialCottrellMixQueue(d_m, 600, 6, .7)
+	allThirty = 1
 	for i in range(10):
 	    b = bcmq.getBatch()
-	    if not len(b)==3:
-		allThree = 0
-	self.failIf(allThree)
+	    if not len(b)==30:
+		allThirty = 0
+	self.failIf(allThirty)
 
 	bcmq.removeAll()
 	bcmq.cleanQueue()
@@ -1442,7 +1519,6 @@
 # LOGGING
 class LogTests(unittest.TestCase):
     def testLogging(self):
-        import cStringIO
         from mixminion.Common import Log, _FileLogHandler, _ConsoleLogHandler
         log = Log("INFO")
         self.assertEquals(log.getMinSeverity(), "INFO")
@@ -1753,8 +1829,8 @@
                 async.process(2)
             
         severity = getLog().getMinSeverity()
-        getLog().setMinSeverity("ERROR") #suppress warning
         try:
+	    suspendLog() # suppress warning
             server.process(0.1)
             t = threading.Thread(None, clientThread)
 
@@ -1763,7 +1839,7 @@
                 server.process(0.1)
             t.join()
         finally:
-            getLog().setMinSeverity(severity) #unsuppress warning
+            resumeLog()  #unsuppress warning
                     
 #----------------------------------------------------------------------
 # Config files
@@ -2242,10 +2318,10 @@
 	self.assertEquals(exampleMod.processedMessages, [])
 	try:
 	    severity = getLog().getMinSeverity()
-	    getLog().setMinSeverity("FATAL") #suppress warning
+	    suspendLog()
 	    manager.sendReadyMessages()
 	finally:
-            getLog().setMinSeverity(severity) #unsuppress warning
+            resumeLog()
 	self.assertEquals(1, queue.count())
 	self.assertEquals(3, len(exampleMod.processedMessages))
 	manager.sendReadyMessages()
@@ -2295,11 +2371,125 @@
 	# FFFF Add tests for catching exceptions from buggy modules
 
 #----------------------------------------------------------------------
+import mixminion.ServerMain
+
+#XXXX DOC
+SERVERCFG = """
+[Server]
+Homedir: %(home)s
+Mode: local
+EncryptIdentityKey: No
+PublicKeyLifetime: 10 days
+IdentityKeyBits: 2048
+EncryptPrivateKey: no
+"""
+
+_FAKE_HOME = None
+def _getKeyring():
+    global _FAKE_HOME
+    if _FAKE_HOME is None:
+	_FAKE_HOME = mix_mktemp()	
+    cfg = SERVERCFG % { 'home' : _FAKE_HOME }
+    conf = mixminion.Config.ServerConfig(string=cfg)
+    return mixminion.ServerMain.ServerKeyring(conf)
+
+_IDENTITY_KEY = None
+def _getIdentityKey():
+    global _IDENTITY_KEY
+    if _IDENTITY_KEY is None:
+	_IDENTITY_KEY = _getKeyring().getIdentityKey()
+    return _IDENTITY_KEY
+
+class ServerMainTests(unittest.TestCase):
+    def testServerKeyring(self):
+	keyring = _getKeyring()
+	home = _FAKE_HOME
+
+	# Test creating identity key
+	identity = _getIdentityKey()
+	fn = os.path.join(home, "keys", "identity.key")
+	identity2 = mixminion.Crypto.pk_PEM_load(fn)
+	self.assertEquals(mixminion.Crypto.pk_get_modulus(identity),
+			  mixminion.Crypto.pk_get_modulus(identity2))
+	# (Make sure warning case can occur.)
+	pk = _ml.rsa_generate(128, 65537)
+	mixminion.Crypto.pk_PEM_save(pk, fn)
+	suspendLog()
+	keyring.getIdentityKey()
+	msg = resumeLog()
+	self.failUnless(len(msg))
+	mixminion.Crypto.pk_PEM_save(identity, fn)
+
+	# Now create a keyset
+	keyring.createKeys(1)
+	# check internal state
+	ivals = keyring.keyIntervals
+	start = mixminion.Common.previousMidnight(time.time())
+	finish = mixminion.Common.previousMidnight(start+(10*24*60*60)+30)
+	self.assertEquals(1, len(ivals))
+	self.assertEquals((start,finish,"0001"), ivals[0])
+
+	keyring.createKeys(2)
+
+	# Check the first key we created
+	va, vu, curKey = keyring._getLiveKey()
+	self.assertEquals(va, start)
+	self.assertEquals(vu, finish)
+	self.assertEquals(vu, keyring.getNextKeyRotation())
+	self.assertEquals(curKey, "0001")
+	keyset = keyring.getServerKeyset()
+	self.assertEquals(keyset.getHashLogFileName(),
+			  os.path.join(home, "work", "hashlogs", "hash_0001"))
+	
+	# Check the second key we created.
+	va, vu, curKey = keyring._getLiveKey(vu + 3600)
+	self.assertEquals(va, finish)
+	self.assertEquals(vu, mixminion.Common.previousMidnight(
+	    finish+10*24*60*60+60))
+
+	# Make a key in the past, to see if it gets scrubbed.
+	keyring.createKeys(1, mixminion.Common.previousMidnight(
+	    start - 10*24*60*60 +60))
+	self.assertEquals(4, len(keyring.keyIntervals))
+	keyring.removeDeadKeys()
+	self.assertEquals(3, len(keyring.keyIntervals))
+	getLog().info("foo")
+	
+	if 0:
+	    # These are slow, since they regenerate the DH params.
+	    # Test getDHFile
+	    f = keyring.getDHFile()
+	    f2 = keyring.getDHFile()
+	    self.assertEquals(f, f2)
+	    
+	    # Test getTLSContext
+	    keyring.getTLSContext()
+
+	# Test getPacketHandler
+	ph = keyring.getPacketHandler()
+
+    def testIncomingQueue(self):
+	# Test deliverMessage.
+	pass
+
+    def testMixPool(self):
+	# Test 'mix' method
+	pass
+
+    def testOutgoingQueue(self):
+	# Test deliverMessage
+	pass
+
+#----------------------------------------------------------------------
 def testSuite():
     suite = unittest.TestSuite()
     loader = unittest.TestLoader()
     tc = loader.loadTestsFromTestCase
 
+    suite.addTest(tc(ServerMainTests))
+    if 0: return suite
+
+    suite.addTest(tc(MiscTests))
     suite.addTest(tc(MinionlibCryptoTests))
     suite.addTest(tc(CryptoTests))
     suite.addTest(tc(PacketTests))

Index: testSupport.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/testSupport.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- testSupport.py	25 Aug 2002 05:58:02 -0000	1.2
+++ testSupport.py	29 Aug 2002 03:30:21 -0000	1.3
@@ -33,18 +33,18 @@
 		   'UseQueue': ('REQUIRE', _parseBoolean, None) } }
     
     def validateConfig(self, sections, entries, lines, contents):
-	loc = sections['Testing/DirectoryDump'].get('Location')
-	if loc and not os.path.isdir(loc):
-	    raise ConfigError("Directory does not exist: %r"%loc)
+	# loc = sections['Testing/DirectoryDump'].get('Location')
+	pass 
+    
     
     def configure(self, config, manager):
-	self.loc = sections['Testing/DirectoryDump'].get('Location')
+	self.loc = config['Testing/DirectoryDump'].get('Location')
 	if not self.loc:
 	    return
-	self.useQueue = sections['Testing/DirectoryDump']['UseQueue']
-	manager.registerModule(self)
+	self.useQueue = config['Testing/DirectoryDump']['UseQueue']
+	#manager.registerModule(self)
 	
-	if not os.path.exits(self.loc):
+	if not os.path.exists(self.loc):
 	    createPrivateDir(self.loc)
 
 	max = -1