[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] The server now runs for me. (Don"t worry -- it will ru...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv8199/lib/mixminion
Modified Files:
Common.py Config.py Crypto.py MMTPServer.py Main.py Modules.py
Queue.py ServerInfo.py ServerMain.py test.py testSupport.py
Log Message:
The server now runs for me. (Don't worry -- it will run for you next.)
I still need to write the client CLI, and test actual delivery before I
can more on to the testing/cleanup phase of 0.0.1.
ServerMain.py:
- Document Keyring and delivery queues
- Add keygen functionality
- Add keygen CLI
- Make key directories if they don't exist
- Generate identity keys as needed
- Check for overlap/gaps in key lifetimes
- Bugfix: Make getLiveKey work
- Resolve typos; clarify names
- Generate keys on startup (temporary measure)
Makefile:
fix openssl url
Common.py:
-Bugfix on log
-Factor out some time-handling functions (mkgmtime, previousMidnight)
Config.py:
- Minor tweaks.
- Add 'IdentityKeyBits' configuration option
Crypto.py:
- Add comments to RNG functions
MMTPServer.py:
- Bugfix: take TLS context as argument, not from config
- Bugfix: Listen on selected IP, not 0.0.0.0
Main.py:
- Add server-keygen command
Modules.py:
- Bugfix: importing from nested package
Queue.py:
- "Cottrell" mixing wasn't really cottrell mixing. Now it is.
ServerInfo.py:
- Spec conformance: key lifetimes must begin at midnight GMT
test.py:
- Factor our supsendLog/resumeLog.
- Add tests for some common stuff
- Add tests for keyrings
testSupport.py:
- Debug.
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Common.py 25 Aug 2002 06:10:33 -0000 1.18
+++ Common.py 29 Aug 2002 03:30:21 -0000 1.19
@@ -307,7 +307,10 @@
def _log(self, severity, message, args):
if _SEVERITIES.get(severity, 100) < self.severity:
return
- m = message % args
+ if args is None:
+ m = message
+ else:
+ m = message % args
for h in self.handlers:
h.write(severity, m)
@@ -337,7 +340,7 @@
indented = "".join(formatted)
if indented.endswith('\n'):
indented = indented[:-1]
- self._log(severity, indented, ())
+ self._log(severity, indented, None)
def error_exc(self, (exclass, ex, tb), message=None, *args):
self.log_exc("ERROR", (exclass, ex, tb), message, *args)
@@ -353,6 +356,23 @@
_THE_LOG = Log('WARN')
return _THE_LOG
+
+#----------------------------------------------------------------------
+# Time processing
+
+def mkgmtime(yyyy,MM,dd,hh,mm,ss):
+ """Analogously to time.mktime, return a number of seconds since the
+ epoch when GMT is yyyy/MM/dd hh:mm:ss"""
+
+ # we set the DST flag to zero so that subtracting time.timezone always
+ # gives us gmt.
+ return time.mktime((yyyy,MM,dd,hh,mm,ss,0,0,0))-time.timezone
+
+def previousMidnight(when):
+ """Given a time_t 'when', return the greatest time_t <= when that falls
+ on midnight, GMT."""
+ yyyy,MM,dd = time.gmtime(when)[0:3]
+ return mkgmtime(yyyy,MM,dd,0,0,0)
#----------------------------------------------------------------------
# Signal handling
Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- Config.py 25 Aug 2002 05:58:02 -0000 1.12
+++ Config.py 29 Aug 2002 03:30:21 -0000 1.13
@@ -124,7 +124,7 @@
def _parseInt(integer):
"""Validation function. Converts a config value to an int.
Raises ConfigError on failure."""
- i = integer.strip().lower()
+ i = integer.strip()
try:
return int(i)
except ValueError, _:
@@ -268,9 +268,7 @@
(0 <= mm < 60) and (0 <= ss <= 61)):
raise ConfigError("Invalid %s %r" % (("date","time")[_timeMode],s))
- # we set the DST flag to zero so that subtracting time.timezone always
- # gives us gmt.
- return time.mktime((yyyy,MM,dd,hh,mm,ss,0,0,0))-time.timezone
+ return mixminion.Common.mkgmtime(yyyy, MM, dd, hh, mm, ss)
def _parseTime(s):
"""Validation function. Converts from DD/MM/YYYY HH:MM:SS format
@@ -644,7 +642,8 @@
'LogLevel' : ('ALLOW', _parseSeverity, "WARN"),
'EchoMessages' : ('ALLOW', _parseBoolean, "no"),
'EncryptIdentityKey' : ('REQUIRE', _parseBoolean, "yes"),
- 'PublicKeyLifetime' : ('REQUIRE', _parseInterval,
+ 'IdentityKeyBits': ('ALLOW', _parseInt, "2048"),
+ 'PublicKeyLifetime' : ('ALLOW', _parseInterval,
"30 days"),
'PublicKeySloppiness': ('ALLOW', _parseInterval,
"5 minutes"),
Index: Crypto.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Crypto.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Crypto.py 25 Aug 2002 06:10:34 -0000 1.18
+++ Crypto.py 29 Aug 2002 03:30:21 -0000 1.19
@@ -427,6 +427,8 @@
# FFFF This implementation is about 2-4x as good as the last one, but
# FFFF still could be better. It's faster than getFloat()*max.
+ # XXXX (This code assumes that integers are at least 32 bits.)
+
assert 0 < max < 0x3fffffff
_ord = ord
while 1:
@@ -440,12 +442,12 @@
return o % max
def getFloat(self):
- """Return a floating-point number between 0 and 1. The number
- will have 'bytes' bytes of resolution."""
+ """Return a floating-point number between 0 and 1."""
b = self.getBytes(4)
_ord = ord
o = ((((((_ord(b[0])&0x7f)<<8) + _ord(b[1]))<<8) +
_ord(b[2]))<<8) + _ord(b[3])
+ #return o / float(0x7fffffff)
return o / 2147483647.0
def _prng(self, n):
Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- MMTPServer.py 25 Aug 2002 05:58:02 -0000 1.14
+++ MMTPServer.py 29 Aug 2002 03:30:21 -0000 1.15
@@ -620,14 +620,14 @@
class MMTPServer(AsyncServer):
"""A helper class to invoke AsyncServer, MMTPServerConnection, and
MMTPClientConnection"""
- def __init__(self, config):
+ def __init__(self, config, tls):
AsyncServer.__init__(self)
- self.context = config.getTLSContext(server=1)
+ self.context = tls
# FFFF Don't always listen; don't always retransmit!
# FFFF Support listening on specific IPs
- self.listener = ListenConnection("0.0.0.0",
- config['Outgoing/MMTP']['Port'],
+ self.listener = ListenConnection(config['Incoming/MMTP']['IP'],
+ config['Incoming/MMTP']['Port'],
LISTEN_BACKLOG,
self._newMMTPConnection)
#self.config = config
Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- Main.py 25 Aug 2002 05:58:02 -0000 1.2
+++ Main.py 29 Aug 2002 03:30:21 -0000 1.3
@@ -75,7 +75,8 @@
_COMMANDS = {
"unittests" : ( 'mixminion.test', 'testAll' ),
"benchmarks" : ( 'mixminion.benchmark', 'timeAll' ),
- "server" : ( 'mixminion.ServerMain', 'runServer' )
+ "server" : ( 'mixminion.ServerMain', 'runServer' ),
+ "server-keygen" : ( 'mixminion.ServerMain', 'runKeygen')
}
def main(args):
Index: Modules.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Modules.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- Modules.py 25 Aug 2002 05:58:02 -0000 1.9
+++ Modules.py 29 Aug 2002 03:30:21 -0000 1.10
@@ -218,7 +218,7 @@
try:
sys.path[0:0] = self.path
try:
- m = __import__(pyPkg, {}, {}, [])
+ m = __import__(pyPkg, {}, {}, [pyClassName])
except ImportError, e:
raise MixError("%s while importing %s" %(str(e),className))
finally:
Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- Queue.py 25 Aug 2002 05:58:02 -0000 1.15
+++ Queue.py 29 Aug 2002 03:30:21 -0000 1.16
@@ -373,19 +373,19 @@
"""A CottrellMixQueue holds a group of files, and returns some of them
as requested, according the Cottrell (timed dynamic-pool) mixing
algorithm from Mixmaster."""
- def __init__(self, location, interval=600, minPoolSize=6, maxSendRate=.3):
+ def __init__(self, location, interval=600, threshold=6, retainRate=.7):
"""Create a new queue that yields a batch of message every 'interval'
- seconds, never allows its pool size to drop below 'minPoolSize',
- and never sends more than maxSendRate * the current pool size."""
+ seconds, never sends unless it has more than <threshold> messages,
+ and always keeps <retainRate> * the current pool size."""
TimedMixQueue.__init__(self, location, interval)
- self.minPoolSize = minPoolSize
- self.maxBatchSize = int(maxSendRate*minPoolSize)
- if self.maxBatchSize < 1:
- self.maxBatchSize = 1
+ self.threshold = threshold
+ self.sendRate = 1.0 - retainRate
def getBatch(self):
pool = self.count()
- nTransmit = min(pool-self.minPoolSize, self.maxBatchSize)
+ if pool <= self.threshold:
+ return []
+ nTransmit = int(pool * self.sendRate)
return self.pickRandom(nTransmit)
class BinomialCottrellMixQueue(CottrellMixQueue):
@@ -393,8 +393,9 @@
from the pool of size P, sends each message with probability N/P."""
def getBatch(self):
pool = self.count()
- nTransmit = min(pool-self.minPoolSize, self.maxBatchSize)
- msgProbability = float(nTransmit) / pool
+ if pool <= self.threshold:
+ return []
+ msgProbability = self.sendRate
return self.rng.shuffle([ h for h in self.getAllMessages()
if self.rng.getFloat() < msgProbability ])
Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- ServerInfo.py 25 Aug 2002 06:10:35 -0000 1.12
+++ ServerInfo.py 29 Aug 2002 03:30:21 -0000 1.13
@@ -250,6 +250,9 @@
if not validAt:
validAt = time.time()
+ # Round validAt to previous mignight.
+ validAt = mixminion.Common.previousMidnight(validAt+30)
+
validUntil = validAt + config['Server']['PublicKeyLifetime'][2]
certStarts = validAt - CERTIFICATE_EXPIRY_SLOPPINESS
certEnds = validUntil + CERTIFICATE_EXPIRY_SLOPPINESS + \
Index: ServerMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerMain.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- ServerMain.py 25 Aug 2002 05:58:02 -0000 1.6
+++ ServerMain.py 29 Aug 2002 03:30:21 -0000 1.7
@@ -14,11 +14,13 @@
import bisect
import mixminion._minionlib
+import mixminion.Crypto
import mixminion.Queue
import mixminion.MMTPServer
-from mixminion.ServerInfo import ServerKeyset, ServerInfo
+from mixminion.ServerInfo import ServerKeyset, ServerInfo, _date, \
+ generateServerDescriptorAndKeys
from mixminion.Common import getLog, MixFatalError, MixError, secureDelete, \
- createPrivateDir
+ createPrivateDir, previousMidnight, ceilDiv
# Directory layout:
# MINION_HOME/work/queues/incoming/
@@ -28,7 +30,8 @@
# tls/dhparam
# hashlogs/hash_1 ...
# log
-# keys/key_1/ServerDesc
+# keys/identity.key
+# key_1/ServerDesc
# mix.key
# mmtp.key
# mmtp.cert
@@ -37,28 +40,60 @@
# ....
class ServerKeyring:
- # homeDir: ----
- # keyDir: ----
- # keySloppiness: ----
- # keyIntervals: list of (start, end, ServerKeyset Name)
+ """A ServerKeyRing remembers current and future keys, descriptors, and
+ hash logs for a mixminion server.
+
+ FFFF: We need a way to generate keys as needed
+ """
+ # homeDir: server home directory
+ # keyDir: server key directory
+ # keySloppiness: fudge-factor: how forgiving are we about key liveness?
+ # keyIntervals: list of (start, end, keyset Name)
+ # liveKey: list of (start, end, keyset name for current key.)
+ # nextRotation: time_t when this key expires.
+ # keyRange: tuple of (firstKey, lastKey) to represent which key names
+ # have keys on disk.
+
def __init__(self, config):
+ "Create a ServerKeyring from a config object"
self.configure(config)
def configure(self, config):
+ "Set up a SeverKeyring from a config object"
+ self.config = config
self.homeDir = config['Server']['Homedir']
self.keyDir = os.path.join(self.homeDir, 'keys')
- self.keySloppiness = config['Server']['PublicKeySloppiness']
+ self.hashDir = os.path.join(self.homeDir, 'work', 'hashlogs')
+ self.keySloppiness = config['Server']['PublicKeySloppiness'][2]
self.checkKeys()
def checkKeys(self):
+ """Internal method: read information about all this server's
+ currently-prepared keys from disk."""
self.keyIntervals = []
+ firstKey = sys.maxint
+ lastKey = 0
+
+ if not os.path.exists(self.keyDir):
+ createPrivateDir(self.keyDir)
+
for dirname in os.listdir(self.keyDir):
+ if not os.path.isdir(os.path.join(self.keyDir,dirname)):
+ continue
if not dirname.startswith('key_'):
getLog().warn("Unexpected directory %s under %s",
dirname, self.keyDir)
continue
keysetname = dirname[4:]
-
+ try:
+ setNum = int(keysetname)
+ if setNum < firstKey: firstKey = setNum
+ if setNum > lastKey: lastKey = setNum
+ except ValueError, _:
+ getLog().warn("Unexpected directory %s under %s",
+ dirname, self.keyDir)
+ continue
+
d = os.path.join(self.keyDir, dirname)
si = os.path.join(d, "ServerDesc")
if os.path.exists(si):
@@ -66,10 +101,94 @@
t1 = inf['Server']['Valid-After']
t2 = inf['Server']['Valid-Until']
self.keyIntervals.append( (t1, t2, keysetname) )
+ else:
+ getLog().warn("No server descriptor found for key %s"%dirname)
self.keyIntervals.sort()
-
+ self.keyRange = (firstKey, lastKey)
+
+ # Now we try to see whether we have more or less than 1 key in effect
+ # for a given time.
+ for idx in xrange(len(self.keyIntervals)-1):
+ end = self.keyIntervals[idx][1]
+ start = self.keyIntervals[idx+1][0]
+ if start < end:
+ getLog().warn("Multiple keys for %s. That's unsupported.",
+ _date(end))
+ elif start > end:
+ getLog().warn("Gap in key schedule: no key from %s to %s",
+ _date(end), _date(start))
+
+ self.nextKeyRotation = 0 # Make sure that now > nextKeyRotation before
+ # we call _getLiveKey()
+ self._getLiveKey() # Set up liveKey, nextKeyRotation.
+
+ def getIdentityKey(self):
+ """Return this server's identity key. Generate one if it doesn't
+ exist."""
+ password = None # FFFF
+ fn = os.path.join(self.keyDir, "identity.key")
+ bits = self.config['Server']['IdentityKeyBits']
+ if os.path.exists(fn):
+ key = mixminion.Crypto.pk_PEM_load(fn, password)
+ keylen = key.get_modulus_bytes()*8
+ if keylen != bits:
+ getLog().warn(
+ "Stored identity key has %s bits, but you asked for %s.",
+ keylen, bits)
+ else:
+ getLog().info("Generating identity key. (This may take a while.)")
+ key = mixminion.Crypto.pk_generate(bits)
+ mixminion.Crypto.pk_PEM_save(key, fn, password)
+ getLog().info("Generated %s-bit identity key.", bits)
+
+ return key
+
+ def createKeys(self, num=1, startAt=None):
+ """Generate 'num' public keys for this server. If startAt is provided,
+ make the first key become valid at'startAt'. Otherwise, make the
+ first key become valid right after the last key we currently have
+ expires. If we have no keys now, make the first key start now."""
+ password = None #FFFF
+
+ if startAt is None:
+ if self.keyIntervals:
+ startAt = self.keyIntervals[-1][1]+60
+ else:
+ startAt = time.time()+60
+
+ startAt = previousMidnight(startAt)
+
+ firstKey, lastKey = self.keyRange
+
+ for i in xrange(num):
+ if firstKey == sys.maxint:
+ keynum = firstkey = lastkey = 1
+ elif firstKey > 1:
+ firstKey -= 1
+ keynum = firstKey
+ else:
+ lastKey += 1
+ keynum = lastKey
+
+ keyname = "%04d" % keynum
+
+ nextStart = startAt + self.config['Server']['PublicKeyLifetime'][2]
+
+ getLog().info("Generating key %s to run from %s through %s",
+ keyname, _date(startAt), _date(nextStart-3600))
+ generateServerDescriptorAndKeys(config=self.config,
+ identityKey=self.getIdentityKey(),
+ keyname=keyname,
+ keydir=self.keyDir,
+ hashdir=self.hashDir,
+ validAt=startAt)
+ startAt = nextStart
+
+ self.checkKeys()
+
def removeDeadKeys(self):
+ """Remove all keys that have expired"""
now = time.time()
cutoff = now - self.keySloppiness
dirs = [ os.path.join(self.keyDir,"key_"+name)
@@ -83,24 +202,44 @@
self.checkKeys()
- def _getLiveKey(self):
- # returns valid-after, valid-until, name
- now = time.time()
- idx = bisect.bisect_left(self.keyIntervals, (now, None, None))
- return self.keyIntervals[idx]
+ def _getLiveKey(self, when=None):
+ """Find the first key that is now valid. Return (Valid-after,
+ valid-util, name)."""
+ if not self.keyIntervals:
+ self.liveKey = None
+ self.nextKeyRotation = 0
+ return None
+
+
+ w = when
+ if when is None:
+ when = time.time()
+ if when < self.nextKeyRotation:
+ return self.liveKey
+
+ idx = bisect.bisect_right(self.keyIntervals, (when, None, None))-1
+ k = self.keyIntervals[idx]
+ if w is None:
+ self.liveKey = k
+ self.nextKeyRotation = k[1]
+
+ return k
def getNextKeyRotation(self):
- return self._getLiveKey()[1]
+ """Return the expiration time of the current key"""
+ return self.nextKeyRotation
def getServerKeyset(self):
+ """Return a ServerKeyset object for the currently live key."""
# FFFF Support passwords on keys
_, _, name = self._getLiveKey()
- hashroot = os.path.join(self.homeDir, 'work', 'hashlogs')
- keyset = ServerKeyset(self.keyDir, name, hashroot)
+ keyset = ServerKeyset(self.keyDir, name, self.hashDir)
keyset.load()
- return self.keyset
+ return keyset
def getDHFile(self):
+ """Return the filename for the diffie-helman parameters for the
+ server. Creates the file if it doesn't yet exist."""
dhdir = os.path.join(self.homeDir, 'work', 'tls')
createPrivateDir(dhdir)
dhfile = os.path.join(dhdir, 'dhparam')
@@ -112,37 +251,48 @@
return dhfile
def getTLSContext(self):
+ """Create and return a TLS context from the currently live key."""
keys = self.getServerKeyset()
return mixminion._minionlib.TLSContext_new(keys.getCertFileName(),
- keys.GetMMTPKey(),
+ keys.getMMTPKey(),
self.getDHFile())
def getPacketHandler(self):
+ """Create and return a PacketHandler from the currently live key."""
keys = self.getServerKeyset()
return mixminion.PacketHandler.PacketHandler(keys.getPacketKey(),
- keys.getHashLogFile())
+ keys.getHashLogFileName())
class IncomingQueue(mixminion.Queue.DeliveryQueue):
+ """A DeliveryQueue to accept messages from incoming MMTP connections,
+ process them with a packet handler, and send them into a mix pool."""
+
def __init__(self, location, packetHandler):
+ """Create an IncomingQueue that stores its messages in <location>
+ and processes them through <packetHandler>."""
mixminion.Queue.DeliveryQueue.__init__(self, location)
self.packetHandler = packetHandler
- self.mixQueue = None
+ self.mixPool = None
- def connectQueues(self, mixQueue):
- self.mixQueue = mixQueue
+ def connectQueues(self, mixPool):
+ """Sets the target mix queue"""
+ self.mixPool = mixPool
def queueMessage(self, msg):
- mixminion.Queue.queueMessage(None, msg)
+ """Add a message for delivery"""
+ mixminion.DeliveryQueue.queueMessage(None, msg)
def deliverMessages(self, msgList):
+ "Implementation of abstract method from DeliveryQueue."
ph = self.packetHandler
for handle, _, message, n_retries in msgList:
try:
res = ph.packetHandler(message)
if res is None:
+ # Drop padding before it gets to the mix.
getLog().info("Padding message dropped")
else:
- self.mixQueue.queueObject(res)
+ self.mixPool.queueObject(res)
self.deliverySucceeded(handle)
except mixminion.Crypto.CryptoError, e:
getLog().warn("Invalid PK or misencrypted packet header:"+str(e))
@@ -154,17 +304,28 @@
getLog().warn("Discarding bad packet:"+str(e))
self.deliveryFailed(handle)
-class MixQueue:
+class MixPool:
+ """Wraps a mixminion.Queue.*MixQueue to send messages to an exit queue
+ and a delivery queue."""
def __init__(self, queue):
+ """Create a new MixPool to wrap a given *MixQueue."""
self.queue = queue
self.outgoingQueue = None
self.moduleManager = None
+ def queueObject(self, obj):
+ """Insert an object into the queue."""
+ self.queue.queueObject(ob)
+
def connectQueues(self, outgoing, manager):
+ """Sets the queue for outgoing mixminion packets, and the
+ module manager for deliverable messages."""
self.outgoingQueue = outgoing
self.moduleManager = manager
def mix(self):
+ """Get a batch of messages, and queue them for delivery as
+ appropriate."""
handles = self.queue.getBatch()
for h in handles:
tp, info = self.queue.getObject(h)
@@ -177,14 +338,20 @@
self.outgoingQueue.queueMessage(ipv4, msg)
class OutgoingQueue(mixminion.Queue.DeliveryQueue):
+ """DeliveryQueue to send messages via outgoing MMTP connections."""
def __init__(self, location):
- OutgoingQueue.__init__(self, location)
+ """Create a new OutgoingQueue that stores its messages in a given
+ location."""
+ mixminion.Queue.DeliveryQueue.__init__(self, location)
self.server = None
def connectQueues(self, server):
+ """Set the MMTPServer that this OutgoingQueue informs of its
+ deliverable messages."""
self.server = server
def deliverMessages(self, msgList):
+ "Implementation of abstract method from DeliveryQueue."
# Map from addr -> [ (handle, msg) ... ]
msgs = {}
for handle, addr, message, n_retries in msgList:
@@ -194,9 +361,11 @@
self.server.sendMessages(addr.ip, addr.port, addr.keyinfo,
messages, handles)
-class _MMTPConnection(mixminion.MMTPServer):
- def __init__(self, config):
- MMTPServer.__init__(self, config)
+class _MMTPServer(mixminion.MMTPServer.MMTPServer):
+ """Implementation of mixminion.MMTPServer that knows about
+ delivery queues."""
+ def __init__(self, config, tls):
+ mixminion.MMTPServer.MMTPServer.__init__(self, config, tls)
def connectQueues(self, incoming, outgoing):
self.incomingQueue = incoming
@@ -211,14 +380,24 @@
def onMessageUndeliverable(self, msg, handle, retriable):
self.outgoingQueue.deliveryFailed(handle, retriable)
-
class MixminionServer:
+ """Wraps and drives all the queues, and the async net server. Handles
+ all timed events."""
def __init__(self, config):
+ """Create a new server from a ServerConfig."""
self.config = config
self.keyring = ServerKeyring(config)
-
+ if self.keyring._getLiveKey() is None:
+ getLog().info("Generating a month's worth of keys.")
+ getLog().info("(Don't count on this feature in future versions.)")
+ # We might not be able to do this, if we password-encrypt keys
+ keylife = config['Server']['PublicKeyLifetime'][2]
+ nKeys = ceilDiv(30*24*60*60, keylife)
+ self.keyring.createKeys(nKeys)
+
self.packetHandler = self.keyring.getPacketHandler()
- self.mmtpConnection = _MMTPConnection(config)
+ tlsContext = self.keyring.getTLSContext()
+ self.mmtpServer = _MMTPServer(config, tlsContext)
# FFFF Modulemanager should know about async so it can patch in if it
# FFFF needs to.
@@ -232,29 +411,32 @@
mixDir = os.path.join(queueDir, "mix")
# FFFF The choice of mix algorithm should be configurable
- self.mixQueue = MixQueue(mixminion.Queue.TimedMixQueue(mixDir, 60))
+ self.mixPool = MixPool(mixminion.Queue.TimedMixQueue(mixDir, 60))
outgoingDir = os.path.join(queueDir, "outgoing")
self.outgoingQueue = OutgoingQueue(outgoingDir)
- self.incomingQueue.connectQueues(mixQueue=self.mixQueue)
- self.mixQueue.connectQueues(outgoing=self.outgoingQueue,
- manager=self.moduleManager)
- self.outgoingQueue.connectQueues(server=self.mmtpConnection)
- self.mmtpConnection.connectQueues(incoming=self.incomingQueue,
- outgoing=self.outgoingQueue)
+ self.incomingQueue.connectQueues(mixPool=self.mixPool)
+ self.mixPool.connectQueues(outgoing=self.outgoingQueue,
+ manager=self.moduleManager)
+ self.outgoingQueue.connectQueues(server=self.mmtpServer)
+ self.mmtpServer.connectQueues(incoming=self.incomingQueue,
+ outgoing=self.outgoingQueue)
def run(self):
+ """Run the server; don't return unless we hit an exception."""
+ # FFFF Use heapq to schedule events?
now = time.time()
nextMix = now + 60 # FFFF Configurable!
nextShred = now + 6000
-
+ nextRotate = self.keyring.getNextKeyRotation() # FFFF use this.
while 1:
while time.time() < nextMix:
- self.mmtpConnection.process(1)
+ self.mmtpServer.process(1)
self.incomingQueue.sendReadyMessages()
- self.mixQueue.mix()
+ getLog().trace("Mix interval elapsed")
+ self.mixPool.mix()
self.outgoingQueue.sendReadyMessages()
self.moduleManager.sendReadyMessages()
@@ -263,7 +445,7 @@
if now > nextShred:
# Configurable shred interval
self.incomingQueue.cleanQueue()
- self.mixQueue.queue.cleanQueue()
+ self.mixPool.queue.cleanQueue()
self.outgoingQueue.cleanQueue()
self.moduleManager.cleanQueues()
nextShred = now + 6000
@@ -276,7 +458,7 @@
print >>sys.stderr, "Usage: %s [-h] [-f configfile]" % cmd
sys.exit(0)
-def configFromArgs(cmd, args):
+def configFromServerArgs(cmd, args):
options, args = getopt.getopt(args, "hf:", ["help", "config="])
if args:
usageAndExit(cmd)
@@ -286,20 +468,23 @@
usageAndExit()
if o in ('-f', '--config'):
configFile = v
+
+ return readConfigFile(configFile)
+
+def readConfigFile(configFile):
try:
- config = mixminion.Config.ServerConfig(fname=configFile)
+ return mixminion.Config.ServerConfig(fname=configFile)
except (IOError, OSError), e:
- print >>sys.stderr, "Error reading configuration file %r"%configFile
+ print >>sys.stderr, "Error reading configuration file %r:"%configFile
+ print >>sys.stderr, " ", str(e)
sys.exit(1)
except mixminion.Config.ConfigError, e:
print >>sys.stderr, "Error in configuration file %r"%configFile
print >>sys.stderr, str(e)
sys.exit(1)
- return config
-
def runServer(cmd, args):
- config = configFromArgs(cmd, args)
+ config = configFromServerArgs(cmd, args)
try:
mixminion.Common.getLog().configure(config)
getLog().debug("Configuring server")
@@ -309,7 +494,7 @@
server = MixminionServer(config)
except:
- getLog().fatal_exc("Exception while configuring server")
+ getLog().fatal_exc(sys.exc_info(),"Exception while configuring server")
print >>sys.stderr, "Shutting down because of exception"
sys.exit(1)
@@ -319,7 +504,40 @@
except KeyboardInterrupt:
pass
except:
- getLog().fatal_exc("Exception while running server")
+ getLog().fatal_exc(sys.exc_info(),"Exception while running server")
getLog().info("Server shutting down")
sys.exit(0)
+
+#----------------------------------------------------------------------
+def runKeygen(cmd, args):
+ options, args = getopt.getopt(args, "hf:n:", ["help", "config=", "keys="])
+ # FFFF password-encrypted keys
+ keys=1
+ usage=0
+ configFile = '/etc/miniond.conf'
+ for opt,val in options:
+ if opt in ('-h', '--help'):
+ usage=1
+ elif opt in ('-f', '--config'):
+ configFile = val
+ elif opt in ('-n', '--keys'):
+ try:
+ keys = int(val)
+ except ValueError, _:
+ print >>sys.stderr,("%s requires an integer" %opt)
+ sys.exit(1)
+ if usage:
+ print >>sys.stderr, "Usage: %s [-h] [-f configfile] [-n nKeys]"%cmd
+ sys.exit(1)
+ config = readConfigFile(configFile)
+
+ getLog().setMinSeverity("INFO")
+ mixminion.Crypto.init_crypto(config)
+ keyring = ServerKeyring(config)
+ print >>sys.stderr, "Creating %s keys..." % keys
+ for i in xrange(keys):
+ keyring.createKeys(1)
+ print >> sys.stderr, ".... (%s/%s done)" % (i+1,keys)
+
+
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- test.py 25 Aug 2002 06:10:35 -0000 1.24
+++ test.py 29 Aug 2002 03:30:21 -0000 1.25
@@ -22,6 +22,7 @@
import base64
import stat
import cPickle
+import cStringIO
from mixminion.testSupport import mix_mktemp
from mixminion.Common import MixError, MixFatalError, MixProtocolError, getLog
@@ -46,6 +47,73 @@
def floatEq(f1,f2):
return abs(f1-f2) < .00001
+def suspendLog():
+ """Temporarily suppress logging output."""
+ log = getLog()
+ if hasattr(log, '_storedHandlers'):
+ resumeLog()
+ buf = cStringIO.StringIO()
+ h = mixminion.Common._ConsoleLogHandler(cStringIO.StringIO())
+ log._storedHandlers = log.handlers
+ log._testBuf = buf
+ log.handlers = []
+ log.addHandler(h)
+
+def resumeLog():
+ """Resume logging output. Return all new log messages since the last
+ suspend."""
+ log = getLog()
+ if not hasattr(log, '_storedHandlers'):
+ return None
+ buf = log._testBuf
+ del log._testBuf
+ log.handlers = log._storedHandlers
+ del log._storedHandlers
+ return str(buf)
+
+#----------------------------------------------------------------------
+# Tests for common functionality
+
+class MiscTests(unittest.TestCase):
+ def testDiv(self):
+ from mixminion.Common import floorDiv, ceilDiv
+
+ self.assertEquals(floorDiv(10,1), 10)
+ self.assertEquals(floorDiv(10,2), 5)
+ self.assertEquals(floorDiv(10,3), 3)
+ self.assertEquals(floorDiv(10,11), 0)
+ self.assertEquals(floorDiv(0,11), 0)
+ self.assertEquals(floorDiv(-1,1), -1)
+ self.assertEquals(floorDiv(-1,2), -1)
+ self.assertEquals(floorDiv(-10,3), -4)
+ self.assertEquals(floorDiv(-10,-3), 3)
+
+ self.assertEquals(ceilDiv(10,1), 10)
+ self.assertEquals(ceilDiv(10,2), 5)
+ self.assertEquals(ceilDiv(10,3), 4)
+ self.assertEquals(ceilDiv(10,11), 1)
+ self.assertEquals(ceilDiv(0,11), 0)
+ self.assertEquals(ceilDiv(-1,1), -1)
+ self.assertEquals(ceilDiv(-1,2), 0)
+ self.assertEquals(ceilDiv(-10,3), -3)
+ self.assertEquals(ceilDiv(-10,-3), 4)
+
+ def testTimeFns(self):
+ from mixminion.Common import floorDiv, mkgmtime, previousMidnight
+ # This isn't a very good test.
+ now = int(time.time())
+ max_sec_per_day = 24*60*60+ 1
+ for t in xrange(10, now, floorDiv(now, 1000)):
+ yyyy,MM,dd,hh,mm,ss = time.gmtime(t)[:6]
+ self.assertEquals(t, mkgmtime(yyyy,MM,dd,hh,mm,ss))
+ pm = previousMidnight(t)
+ yyyy2,MM2,dd2,hh2,mm2,ss2 = time.gmtime(pm)[:6]
+ self.assertEquals((yyyy2,MM2,dd2), (yyyy,MM,dd))
+ self.assertEquals((0,0,0), (hh2,mm2,ss2))
+ self.failUnless(pm <= t and 0 <= (t-pm) <= max_sec_per_day)
+ self.assertEquals(previousMidnight(t), pm)
+ self.assertEquals(previousMidnight(pm), pm)
+
#----------------------------------------------------------------------
import mixminion._minionlib as _ml
@@ -367,7 +435,7 @@
def test_aesprng(self):
# Make sure that AESCounterPRNG is really repeatable.
- key ="aaaa"*4
+ key ="aaab"*4
PRNG = AESCounterPRNG(key)
self.assert_(prng(key,100000) == (
PRNG.getBytes(5)+PRNG.getBytes(16*1024-5)+
@@ -380,6 +448,15 @@
for i in xrange(1,10000,17):
self.failUnless(0 <= PRNG.getInt(10) < 10)
self.failUnless(0 <= PRNG.getInt(i) < i)
+
+## itot=ftot=0
+## for i in xrange(1000000):
+## itot += PRNG.getInt(10)
+## ftot += PRNG.getFloat()
+
+## print "AVG INT", itot/1000000.0
+## print "AVG FLT", ftot/1000000.0
+
for i in xrange(100):
self.failUnless(0 <= PRNG.getFloat() < 1)
@@ -1403,8 +1480,8 @@
b.sort()
self.assertEquals(msgs,b)
- cmq = CottrellMixQueue(d_m, 600, 6, .5)
- # Not enough messages
+ cmq = CottrellMixQueue(d_m, 600, 6, .7)
+ # Not enough messages (<= 6)
self.assertEquals([], cmq.getBatch())
self.assertEquals([], cmq.getBatch())
# 8 messages: 2 get sent
@@ -1421,19 +1498,19 @@
if b != b1:
allEq = 0; break
self.failIf(allEq)
- # Don't send more than 3.
- for x in xrange(100):
+ # Send 30 when there are 100 messages.
+ for x in xrange(92):
cmq.queueMessage("Hello2 %s"%x)
for x in xrange(10):
- self.assertEquals(3, len(cmq.getBatch()))
+ self.assertEquals(30, len(cmq.getBatch()))
- bcmq = BinomialCottrellMixQueue(d_m, 600, 6, .5)
- allThree = 1
+ bcmq = BinomialCottrellMixQueue(d_m, 600, 6, .7)
+ allThirty = 1
for i in range(10):
b = bcmq.getBatch()
- if not len(b)==3:
- allThree = 0
- self.failIf(allThree)
+ if not len(b)==30:
+ allThirty = 0
+ self.failIf(allThirty)
bcmq.removeAll()
bcmq.cleanQueue()
@@ -1442,7 +1519,6 @@
# LOGGING
class LogTests(unittest.TestCase):
def testLogging(self):
- import cStringIO
from mixminion.Common import Log, _FileLogHandler, _ConsoleLogHandler
log = Log("INFO")
self.assertEquals(log.getMinSeverity(), "INFO")
@@ -1753,8 +1829,8 @@
async.process(2)
severity = getLog().getMinSeverity()
- getLog().setMinSeverity("ERROR") #suppress warning
try:
+ suspendLog() # suppress warning
server.process(0.1)
t = threading.Thread(None, clientThread)
@@ -1763,7 +1839,7 @@
server.process(0.1)
t.join()
finally:
- getLog().setMinSeverity(severity) #unsuppress warning
+ resumeLog() #unsuppress warning
#----------------------------------------------------------------------
# Config files
@@ -2242,10 +2318,10 @@
self.assertEquals(exampleMod.processedMessages, [])
try:
severity = getLog().getMinSeverity()
- getLog().setMinSeverity("FATAL") #suppress warning
+ suspendLog()
manager.sendReadyMessages()
finally:
- getLog().setMinSeverity(severity) #unsuppress warning
+ resumeLog()
self.assertEquals(1, queue.count())
self.assertEquals(3, len(exampleMod.processedMessages))
manager.sendReadyMessages()
@@ -2295,11 +2371,125 @@
# FFFF Add tests for catching exceptions from buggy modules
#----------------------------------------------------------------------
+import mixminion.ServerMain
+
+#XXXX DOC
+SERVERCFG = """
+[Server]
+Homedir: %(home)s
+Mode: local
+EncryptIdentityKey: No
+PublicKeyLifetime: 10 days
+IdentityKeyBits: 2048
+EncryptPrivateKey: no
+"""
+
+_FAKE_HOME = None
+def _getKeyring():
+ global _FAKE_HOME
+ if _FAKE_HOME is None:
+ _FAKE_HOME = mix_mktemp()
+ cfg = SERVERCFG % { 'home' : _FAKE_HOME }
+ conf = mixminion.Config.ServerConfig(string=cfg)
+ return mixminion.ServerMain.ServerKeyring(conf)
+
+_IDENTITY_KEY = None
+def _getIdentityKey():
+ global _IDENTITY_KEY
+ if _IDENTITY_KEY is None:
+ _IDENTITY_KEY = _getKeyring().getIdentityKey()
+ return _IDENTITY_KEY
+
+class ServerMainTests(unittest.TestCase):
+ def testServerKeyring(self):
+ keyring = _getKeyring()
+ home = _FAKE_HOME
+
+ # Test creating identity key
+ identity = _getIdentityKey()
+ fn = os.path.join(home, "keys", "identity.key")
+ identity2 = mixminion.Crypto.pk_PEM_load(fn)
+ self.assertEquals(mixminion.Crypto.pk_get_modulus(identity),
+ mixminion.Crypto.pk_get_modulus(identity2))
+ # (Make sure warning case can occur.)
+ pk = _ml.rsa_generate(128, 65537)
+ mixminion.Crypto.pk_PEM_save(pk, fn)
+ suspendLog()
+ keyring.getIdentityKey()
+ msg = resumeLog()
+ self.failUnless(len(msg))
+ mixminion.Crypto.pk_PEM_save(identity, fn)
+
+ # Now create a keyset
+ keyring.createKeys(1)
+ # check internal state
+ ivals = keyring.keyIntervals
+ start = mixminion.Common.previousMidnight(time.time())
+ finish = mixminion.Common.previousMidnight(start+(10*24*60*60)+30)
+ self.assertEquals(1, len(ivals))
+ self.assertEquals((start,finish,"0001"), ivals[0])
+
+ keyring.createKeys(2)
+
+ # Check the first key we created
+ va, vu, curKey = keyring._getLiveKey()
+ self.assertEquals(va, start)
+ self.assertEquals(vu, finish)
+ self.assertEquals(vu, keyring.getNextKeyRotation())
+ self.assertEquals(curKey, "0001")
+ keyset = keyring.getServerKeyset()
+ self.assertEquals(keyset.getHashLogFileName(),
+ os.path.join(home, "work", "hashlogs", "hash_0001"))
+
+ # Check the second key we created.
+ va, vu, curKey = keyring._getLiveKey(vu + 3600)
+ self.assertEquals(va, finish)
+ self.assertEquals(vu, mixminion.Common.previousMidnight(
+ finish+10*24*60*60+60))
+
+ # Make a key in the past, to see if it gets scrubbed.
+ keyring.createKeys(1, mixminion.Common.previousMidnight(
+ start - 10*24*60*60 +60))
+ self.assertEquals(4, len(keyring.keyIntervals))
+ keyring.removeDeadKeys()
+ self.assertEquals(3, len(keyring.keyIntervals))
+ getLog().info("foo")
+
+ if 0:
+ # These are slow, since they regenerate the DH params.
+ # Test getDHFile
+ f = keyring.getDHFile()
+ f2 = keyring.getDHFile()
+ self.assertEquals(f, f2)
+
+ # Test getTLSContext
+ keyring.getTLSContext()
+
+ # Test getPacketHandler
+ ph = keyring.getPacketHandler()
+
+ def testIncomingQueue(self):
+ # Test deliverMessage.
+ pass
+
+ def testMixPool(self):
+ # Test 'mix' method
+ pass
+
+ def testOutgoingQueue(self):
+ # Test deliverMessage
+ pass
+
+#----------------------------------------------------------------------
def testSuite():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
tc = loader.loadTestsFromTestCase
+ suite.addTest(tc(ServerMainTests))
+ if 0: return suite
+
+ suite.addTest(tc(MiscTests))
suite.addTest(tc(MinionlibCryptoTests))
suite.addTest(tc(CryptoTests))
suite.addTest(tc(PacketTests))
Index: testSupport.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/testSupport.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- testSupport.py 25 Aug 2002 05:58:02 -0000 1.2
+++ testSupport.py 29 Aug 2002 03:30:21 -0000 1.3
@@ -33,18 +33,18 @@
'UseQueue': ('REQUIRE', _parseBoolean, None) } }
def validateConfig(self, sections, entries, lines, contents):
- loc = sections['Testing/DirectoryDump'].get('Location')
- if loc and not os.path.isdir(loc):
- raise ConfigError("Directory does not exist: %r"%loc)
+ # loc = sections['Testing/DirectoryDump'].get('Location')
+ pass
+
def configure(self, config, manager):
- self.loc = sections['Testing/DirectoryDump'].get('Location')
+ self.loc = config['Testing/DirectoryDump'].get('Location')
if not self.loc:
return
- self.useQueue = sections['Testing/DirectoryDump']['UseQueue']
- manager.registerModule(self)
+ self.useQueue = config['Testing/DirectoryDump']['UseQueue']
+ #manager.registerModule(self)
- if not os.path.exits(self.loc):
+ if not os.path.exists(self.loc):
createPrivateDir(self.loc)
max = -1