[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Document all the source code; refactor a little.
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv27654/lib/mixminion
Modified Files:
BuildMessage.py ClientMain.py Common.py Config.py Crypto.py
HashLog.py MMTPClient.py MMTPServer.py Main.py Modules.py
Packet.py PacketHandler.py Queue.py ServerInfo.py
ServerMain.py __init__.py benchmark.py test.py testSupport.py
Log Message:
Document all the source code; refactor a little.
All Python code:
Lots of documentation and comments.
Refactor getLog() method into a LOG constant.
Be less verbose when catching (but not binding) exceptions. Apparently,
I had misremembered the syntax that lets you say:
1) try...except:
2) try...except type:
3) try...except type, var:
4) try...except (type,type,type):
5) try...except (type,type,type), var:
but *not*:
6) try...except var:
as allowing 6, but forbidding 2 and 4. That's wrong.
ClientMain:
Move file support from getPath into getServerInfo.
Common:
Don't try to call shred with more than 250 args at a time; cope with
old or broken unices.
Always use the backward-compatible version of _logtime. Yes, the
simpler one is 27% faster, but if we're spewing log messages fast
enough for that to matter, we're spewing too much.
Config:
Reject config files where header lines are indented
Use tuples for section entries, not lists. This should be faster in
nearly all cases.
Crypto:
Use a faster lioness_decrypt (by about 10%)
Inline _trng_uncached.
MMTPClient:
Make .shutdown() work even when .connect() has failed.
Main:
There are mumblings on python-dev that sys.path may eventually include
non-strings. Thus, let's give users with special import handlers
the benefit of the doubt.
MMTPServer, ServerName:
Rename MMTPServer
testSupport, test:
Speed up tests by including a few built-in 2048bit RSA keys in testSupport.
test:
Improve various tests
Test journaling correctness on HashLog
Better tests for BinomialCottrelMixQueue
Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- BuildMessage.py 7 Dec 2002 04:03:35 -0000 1.19
+++ BuildMessage.py 9 Dec 2002 04:47:39 -0000 1.20
@@ -9,7 +9,7 @@
import zlib
import operator
from mixminion.Packet import *
-from mixminion.Common import MixError, MixFatalError, getLog
+from mixminion.Common import MixError, MixFatalError, LOG
import mixminion.Crypto as Crypto
import mixminion.Modules as Modules
@@ -33,11 +33,11 @@
if paddingPRNG is None: paddingPRNG = Crypto.AESCounterPRNG()
assert path1 and path2
- getLog().debug("Encoding forward message for %s-byte payload",len(payload))
- getLog().debug(" Using path %s/%s",
+ LOG.debug("Encoding forward message for %s-byte payload",len(payload))
+ LOG.debug(" Using path %s/%s",
[s.getNickname() for s in path1],
[s.getNickname() for s in path2])
- getLog().debug(" Delivering to %04x:%r", exitType, exitInfo)
+ LOG.debug(" Delivering to %04x:%r", exitType, exitInfo)
# Compress, pad, and checksum the payload.
payload = _encodePayload(payload, 0, paddingPRNG)
@@ -64,12 +64,12 @@
if paddingPRNG is None: paddingPRNG = Crypto.AESCounterPRNG()
if secretRNG is None: secretRNG = paddingPRNG
- getLog().debug("Encoding encrypted forward message for %s-byte payload",
+ LOG.debug("Encoding encrypted forward message for %s-byte payload",
len(payload))
- getLog().debug(" Using path %s/%s",
+ LOG.debug(" Using path %s/%s",
[s.getNickname() for s in path1],
[s.getNickname() for s in path2])
- getLog().debug(" Delivering to %04x:%r", exitType, exitInfo)
+ LOG.debug(" Delivering to %04x:%r", exitType, exitInfo)
# Compress, pad, and checksum the payload.
# (For encrypted-forward messages, we have overhead for OAEP padding
@@ -115,9 +115,9 @@
"""
if paddingPRNG is None: paddingPRNG = Crypto.AESCounterPRNG()
- getLog().debug("Encoding reply message for %s-byte payload",
+ LOG.debug("Encoding reply message for %s-byte payload",
len(payload))
- getLog().debug(" Using path %s/??",[s.getNickname() for s in path1])
+ LOG.debug(" Using path %s/??",[s.getNickname() for s in path1])
# Compress, pad, and checksum the payload.
payload = _encodePayload(payload, 0, paddingPRNG)
@@ -154,9 +154,9 @@
if secretPRNG is None:
secretPRNG = Crypto.AESCounterPRNG()
- getLog().debug("Building reply block for path %s",
+ LOG.debug("Building reply block for path %s",
[s.getNickname() for s in path])
- getLog().debug(" Delivering to %04x:%r", exitType, exitInfo)
+ LOG.debug(" Delivering to %04x:%r", exitType, exitInfo)
# The message is encrypted first by the end-to-end key, then by
# each of the path keys in order. We need to reverse these steps, so we
@@ -251,7 +251,7 @@
if Crypto.sha1(tag+userKey+"Validate")[-1] == '\x00':
try:
return _decodeStatelessReplyPayload(payload, tag, userKey)
- except MixError, _:
+ except MixError:
pass
# If we have an RSA key, and none of the above steps get us a good
@@ -284,7 +284,7 @@
msg = tag+payload
try:
rsaPart = Crypto.pk_decrypt(msg[:key.get_modulus_bytes()], key)
- except Crypto.CryptoError, _:
+ except Crypto.CryptoError:
return None
rest = msg[key.get_modulus_bytes():]
# XXXX001 magic string
@@ -626,7 +626,7 @@
if nil != '':
raise ParseError("Error in compressed data")
return d
- except zlib.error, _:
+ except zlib.error:
raise ParseError("Error in compressed data")
def _validateZlib():
@@ -645,7 +645,7 @@
_ZLIB_LIBRARY_OK = 1
return
- getLog().warn("Unrecognized zlib version: %r. Spot-checking output", ver)
+ LOG.warn("Unrecognized zlib version: %r. Spot-checking output", ver)
# This test is inadequate, but it _might_ catch future incompatible
# changes.
_ZLIB_LIBRARY_OK = 0.5
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- ClientMain.py 7 Dec 2002 04:03:35 -0000 1.10
+++ ClientMain.py 9 Dec 2002 04:47:39 -0000 1.11
@@ -31,7 +31,7 @@
import time
import types
-from mixminion.Common import getLog, floorDiv, createPrivateDir, MixError, \
+from mixminion.Common import LOG, floorDiv, createPrivateDir, MixError, \
MixFatalError
import mixminion.Crypto
import mixminion.BuildMessage
@@ -43,9 +43,27 @@
from mixminion.Modules import MBOX_TYPE, SMTP_TYPE, DROP_TYPE
class TrivialKeystore:
- '''This is a temporary keystore implementation until we get a working
- directory server implementation.'''
+ """This is a temporary keystore implementation until we get a working
+ directory server implementation.
+
+ The idea couldn't be simpler: we just keep a directory of files, each
+ containing a single server descriptor. We cache nothing; we validate
+ everything; we have no automatic path generation. Servers can be
+ accessed by nickname, by filename within our directory, or by filename
+ from elsewhere on the filesystem.
+
+ We skip all server descriptors that have expired, that will
+ soon expire, or which aren't yet in effect.
+ """
+ ## Fields:
+ # directory: path to the directory we scan for server descriptors.
+ # byNickname: a map from nickname to valid ServerInfo object.
+ # byFilename: a map from filename within self.directory to valid
+ # ServerInfo object.
def __init__(self, directory, now=None):
+ """Create a new TrivialKeystore to access the descriptors stored in
+ directory. Selects descriptors that are valid at the time 'now',
+ or at the current time if 'now' is None."""
self.directory = directory
createPrivateDir(directory)
self.byNickname = {}
@@ -55,52 +73,77 @@
now = time.time()
for f in os.listdir(self.directory):
+ # Try to read a file: is it a server descriptor?
p = os.path.join(self.directory, f)
try:
info = ServerInfo(fname=p, assumeValid=0)
- except ConfigError, _:
- getLog().warn("Invalid server descriptor %s", p)
+ except ConfigError:
+ LOG.warn("Invalid server descriptor %s", p)
continue
+ # Find its nickname and normalized filename
serverSection = info['Server']
nickname = serverSection['Nickname']
if '.' in f:
f = f[:f.rindex('.')]
+ # Skip the descriptor if it isn't valid yet...
if now < serverSection['Valid-After']:
- getLog().info("Ignoring future decriptor %s", p)
+ LOG.info("Ignoring future decriptor %s", p)
continue
+ # ... or if it's expired ...
if now >= serverSection['Valid-Until']:
- getLog().info("Ignoring expired decriptor %s", p)
+ LOG.info("Ignoring expired decriptor %s", p)
continue
+ # ... or if it's going to expire within 3 hours (HACK!).
if now + 3*60*60 >= serverSection['Valid-Until']:
- getLog().info("Ignoring soon-to-expire decriptor %s", p)
+ LOG.info("Ignoring soon-to-expire decriptor %s", p)
continue
+ # Only allow one server per nickname ...
if self.byNickname.has_key(nickname):
- getLog().warn(
+ LOG.warn(
"Ignoring descriptor %s with duplicate nickname %s",
p, nickname)
continue
+ # ... and per normalized filename.
if self.byFilename.has_key(f):
- getLog().warn(
+ LOG.warn(
"Ignoring descriptor %s with duplicate prefix %s",
p, f)
continue
- getLog().info("Loaded server %s from %s", nickname, f)
+ LOG.info("Loaded server %s from %s", nickname, f)
+ # Okay, it's good. Cache it.
self.byNickname[nickname] = info
self.byFilename[f] = info
def getServerInfo(self, name):
+ """Return a ServerInfo object corresponding to 'name'. If 'name' is
+ a ServerInfo object, returns 'name'. Otherwise, checks server by
+ nickname, then by filename within the keystore, then by filename
+ on the file system. If no server is found, returns None."""
if isinstance(name, ServerInfo):
return name
- if self.byNickname.has_key(name):
+ elif self.byNickname.has_key(name):
return self.byNickname[name]
- if self.byFilename.has_key(name):
+ elif self.byFilename.has_key(name):
return self.byFilename[name]
- return None
+ elif os.path.exists(name):
+ try:
+ return ServerInfo(fname=name, assumeValid=0)
+ except OSError, e:
+ raise MixError("Couldn't read descriptor %s: %s" %
+ (s, e))
+ except ConfigError, e:
+ raise MixError("Couldn't parse descriptor %s: %s" %
+ (s, e))
+ else:
+ return None
def getPath(self, serverList):
+ """Given a sequence of strings of ServerInfo objects, resolves each
+ one according to the rule of getServerInfo, and returns a list of
+ ServerInfos. Raises MixError if any server can't be resolved."""
path = []
for s in serverList:
if isinstance(s, ServerInfo):
@@ -109,29 +152,23 @@
server = self.getServerInfo(s)
if server is not None:
path.append(server)
- elif os.path.exists(s):
- try:
- server = ServerInfo(fname=s, assumeValid=0)
- path.append(server)
- except OSError, e:
- raise MixError("Couldn't read descriptor %s: %s" %
- (s, e))
- except ConfigError, e:
- raise MixError("Couldn't parse descriptor %s: %s" %
- (s, e))
else:
raise MixError("Couldn't find descriptor %s" % s)
return path
def getRandomServers(self, prng, n):
+ """Returns a list of n different servers, in random order, according
+ to prng. Raises MixError if not enough exist.
+
+ (This isn't actually used.)"""
vals = self.byNickname.values()
if len(vals) < n:
- raise MixFatalError("Not enough servers (%s requested)", n)
+ raise MixError("Not enough servers (%s requested)", n)
return prng.shuffle(vals, n)
def installDefaultConfig(fname):
"""Create a default, 'fail-safe' configuration in a given file"""
- getLog().warn("No configuration file found. Installing default file in %s",
+ LOG.warn("No configuration file found. Installing default file in %s",
fname)
f = open(os.path.expanduser(fname), 'w')
f.write("""\
@@ -161,7 +198,14 @@
f.close()
class MixminionClient:
+ """Access point for client functionality. Currently, this is limited
+ to generating and sending forward messages"""
+ ## Fields:
+ # config: The ClientConfig object with the current configuration
+ # keystore: A TrivialKeystore object
+ # prng: A pseudo-random number generator for padding and path selection
def __init__(self, conf):
+ """Create a new MixminionClient with a given configuration"""
self.config = conf
# Make directories
@@ -178,13 +222,27 @@
self.prng = mixminion.Crypto.AESCounterPRNG()
def sendForwardMessage(self, address, payload, path1, path2):
+ """Generate and send a forward message.
+ address -- the results of a parseAddress call
+ payload -- the contents of the message to send
+ path1,path2 -- lists of servers or server names for the first and
+ second legs of the path, respectively. These are processed
+ as described in TrivialKeystore.getServerInfo"""
message, firstHop = \
self.generateForwardMessage(address, payload, path1, path2)
self.sendMessages([message], firstHop)
def generateForwardMessage(self, address, payload, path1, path2):
- if not path1:
+ """Generate a forward message, but do not send it. Returns
+ a tuple of (the message body, a ServerInfo for the first hop.)
+
+ address -- the results of a parseAddress call
+ payload -- the contents of the message to send
+ path1,path2 -- lists of servers or server names for the first and
+ second legs of the path, respectively. These are processed
+ as described in TrivialKeystore.getServerInfo"""
+ if not path1:
raise MixError("No servers in first leg of path")
if not path2:
raise MixError("No servers in second leg of path")
@@ -208,10 +266,13 @@
else:
servers2.append(self.keystore.getServerInfo(lastHop))
msg = mixminion.BuildMessage.buildForwardMessage(
- payload, routingType, routingInfo, servers1, servers2)
+ payload, routingType, routingInfo, servers1, servers2,
+ self.prng)
return msg, servers1[0]
def sendMessages(self, msgList, server):
+ """Given a list of packets and a ServerInfo object, sends the
+ packets to the server via MMTP"""
con = mixminion.MMTPClient.BlockingClientConnection(server.getAddr(),
server.getPort(),
server.getKeyID())
@@ -223,10 +284,16 @@
con.shutdown()
def parseAddress(s):
- """DOCDOC
- format is mbox:name@server OR [smtp:]mailbox OR drop OR test:rinfo
- or 0xABCD:address """
- # DOCDOC
+ """Parse and validate an address; takes a string, and returns an Address
+ object.
+
+ Accepts strings of the format:
+ mbox:<mailboxname>@<server>
+ OR smtp:<email address>
+ OR <email address> (smtp is implicit)
+ OR drop
+ OR 0x<routing type>:<routing info>
+ """
# ???? Should this should get refactored into clientmodules, or someplace?
if s.lower() == 'drop':
return Address(DROP_TYPE, None, None)
@@ -235,14 +302,14 @@
elif ':' not in s:
try:
return Address(SMTP_TYPE, parseSMTPInfo(s).pack(), None)
- except ParseError, _:
+ except ParseError:
raise ParseError("Can't parse address %s"%s)
tp,val = s.split(':', 1)
tp = tp.lower()
if tp.startswith("0x"):
try:
tp = int(tp[2:], 16)
- except ValueError, _:
+ except ValueError:
raise ParseError("Invalid hexidecimal value %s"%tp)
if not (0x0000 <= tp <= 0xFFFF):
raise ParseError("Invalid type: 0x%04x"%tp)
@@ -262,6 +329,10 @@
raise ParseError("Unrecognized address type: %s"%s)
class Address:
+ """Represents the target address for a Mixminion message.
+ Consists of the exitType for the final hop, the routingInfo for
+ the last hop, and (optionally) a server to use as the last hop.
+ """
def __init__(self, exitType, exitAddress, lastHop=None):
self.exitType = exitType
self.exitAddress = exitAddress
@@ -339,11 +410,11 @@
installDefaultConfig(configFile)
config = readConfigFile(configFile)
- getLog().configure(config)
+ LOG.configure(config)
if verbose:
- getLog().setMinSeverity("DEBUG")
+ LOG.setMinSeverity("DEBUG")
- getLog().debug("Configuring client")
+ LOG.debug("Configuring client")
mixminion.Common.configureShredCommand(config)
mixminion.Crypto.init_crypto(config)
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- Common.py 7 Dec 2002 04:03:35 -0000 1.29
+++ Common.py 9 Dec 2002 04:47:39 -0000 1.30
@@ -7,7 +7,7 @@
__all__ = [ 'MixError', 'MixFatalError', 'onReset', 'onTerminate',
'installSignalHandlers', 'secureDelete', 'secureRename',
- 'ceilDiv', 'floorDiv', 'getLog', 'stringContains' ]
+ 'ceilDiv', 'floorDiv', 'LOG', 'stringContains' ]
import os
import signal
@@ -75,7 +75,7 @@
raise MixFatalError("Nonexistent directory %s" % d)
try:
os.makedirs(d, 0700)
- except OSError, _:
+ except OSError:
raise MixFatalError("Unable to create directory %s" % d)
checkPrivateDir(d)
@@ -121,13 +121,17 @@
if (mode & 020) and not (mode & stat.S_ISVTX):
# FFFF We may want to give an even stronger error here.
- getLog().warn("Iffy mode %o on directory %s (Writable by gid %s)",
- mode, d, st[stat.ST_GID])
+ LOG.warn("Iffy mode %o on directory %s (Writable by gid %s)",
+ mode, d, st[stat.ST_GID])
#----------------------------------------------------------------------
# Secure filesystem operations.
-#
+
+# A 'shred' command to overwrite and unlink files. It should accept an
+# arbitrary number of arguments. (If "---", we haven't configured the
+# shred command. If None, we're using our internal implementation.)
_SHRED_CMD = "---"
+# Tuple of options to be passed to the 'shred' command
_SHRED_OPTS = None
def configureShredCommand(conf):
@@ -145,13 +149,14 @@
if os.path.exists("/usr/bin/shred"):
cmd, opts = "/usr/bin/shred", ["-uz", "-n0"]
else:
- getLog().warn("Files will not be securely deleted.")
+ LOG.warn("Files will not be securely deleted.")
cmd, opts = None, None
_SHRED_CMD, _SHRED_OPTS = cmd, opts
-# Size of a block on the filesystem we're overwriting on
+# Size of a block on the filesystem we're overwriting on; If zero, we need
+# to determine it.
_BLKSIZE = 0
# A string of _BLKSIZE zeros
_NILSTR = None
@@ -181,9 +186,7 @@
"""Given a list of filenames, removes the contents of all of those
files, from the disk, 'securely'. If blocking=1, does not
return until the remove is complete. If blocking=0, returns
- immediately, and returns the PID of the process removing the
- files. (Returns None if this process unlinked the files
- itself.)
+ immediately, and continues removing the files in the background.
Securely deleting files only does so much good. Metadata on
the file system, such as atime and dtime, can still be used to
@@ -230,7 +233,10 @@
else:
mode = os.P_NOWAIT
- return os.spawnl(mode, _SHRED_CMD, _SHRED_CMD, *(_SHRED_OPTS+fnames))
+ # Some systems are unhappy when you call them with too many options.
+ for i in xrange(0, len(fnames), 250-len(_SHRED_OPTS)):
+ files = fnames[i:i+250-len(_SHRED_OPTS)]
+ os.spawnl(mode, _SHRED_CMD, _SHRED_CMD, *(_SHRED_OPTS+files))
#----------------------------------------------------------------------
# Logging
@@ -238,29 +244,24 @@
# I'm trying to make this interface look like a subset of the one in
# the draft PEP-0282 (http://www.python.org/peps/pep-0282.html).
-if sys.version_info[:2] >= (2,1):
- def _logtime():
- 'Helper function. Returns current local time formatted for log.'
-
- # Note: Python strftime is implemented using that platform libc's
- # strftime, so in theory, this might barf. All of the format
- # elements below are (I think) standard, so we should be ok.
- return time.strftime("%b %d %H:%M:%S")
-else:
- def _logtime():
- 'Helper function. Returns current local time formatted for log.'
- return time.strftime("%b %d %H:%M:%S", time.localtime(time.time()))
+def _logtime():
+ 'Helper function. Returns current local time formatted for log.'
+ return time.strftime("%b %d %H:%M:%S", time.localtime(time.time()))
class _FileLogHandler:
"""Helper class for logging. Represents a file on disk, and allows the
usual close-and-open gimmick for log rotation."""
+ ## Fields:
+ # file -- a file object, or None if the file is closed.
+ # fname -- this log's associated filename
def __init__(self, fname):
"Create a new FileLogHandler to append messages to fname"
self.file = None
self.fname = fname
self.reset()
def reset(self):
- "Close and reopen our underlying file"
+ """Close and reopen our underlying file. This behavior is needed
+ to implement log rotation."""
if self.file is not None:
self.file.close()
try:
@@ -268,25 +269,31 @@
if not os.path.exists(parent):
createPrivateDir(parent)
self.file = open(self.fname, 'a')
- except OSError, _:
+ except OSError:
self.file = None
raise MixError("Unable to open log file %r"%self.fname)
def close(self):
"Close the underlying file"
self.file.close()
def write(self, severity, message):
+ """(Used by Log: write a message to this log handler.)"""
if self.file is None:
return
print >> self.file, "%s [%s] %s" % (_logtime(), severity, message)
class _ConsoleLogHandler:
+ """Helper class for logging: directs all log messages to a stderr-like
+ file object"""
def __init__(self, file):
+ "Create a new _ConsoleLogHandler attached to a given file."""
self.file = file
def reset(self): pass
def close(self): pass
def write(self, severity, message):
+ """(Used by Log: write a message to this log handler.)"""
print >> self.file, "%s [%s] %s" % (_logtime(), severity, message)
+# Map from log severity name to numeric values
_SEVERITIES = { 'TRACE' : -2,
'DEBUG' : -1,
'INFO' : 0,
@@ -308,12 +315,22 @@
WARN: recoverable errors
ERROR: nonrecoverable errors that affect only a single
message or a connection.
- FATAL: nonrecoverable errors that affect the entire system"""
+ FATAL: nonrecoverable errors that affect the entire system.
+
+ In practise, we instantiate only a single instance of this class,
+ accessed as mixminion.Common.LOG."""
+ ## Fields:
+ # handlers: a list of logHandler objects.
+ # severity: a severity below which log messages are ignored.
def __init__(self, minSeverity):
+ """Create a new Log object that ignores all message less severe than
+ minSeverity, and sends its output to stderr."""
self.configure(None)
self.setMinSeverity(minSeverity)
def configure(self, config):
+ """Set up this Log object based on a ServerConfig or ClientConfig
+ object"""
self.handlers = []
if config == None or not config.has_section("Server"):
self.setMinSeverity("WARN")
@@ -335,18 +352,25 @@
del self.handlers[0]
def setMinSeverity(self, minSeverity):
+ """Sets the minimum severity of messages to be logged.
+ minSeverity -- the string representation of a severity level."""
self.severity = _SEVERITIES.get(minSeverity, 1)
def getMinSeverity(self):
+ """Return a string representation of this log's minimum severity
+ level."""
for k,v in _SEVERITIES.items():
if v == self.severity:
return k
- return _SEVERITIES['INFO']
+ return "INFO"
def addHandler(self, handler):
+ """Add a LogHandler object to the list of objects that receive
+ messages from this log."""
self.handlers.append(handler)
def reset(self):
+ """Flush and re-open all logs."""
for h in self.handlers:
try:
h.reset()
@@ -357,13 +381,19 @@
print >>sys.stderr, "Unable to reset log system"
def close(self):
+ """Close all logs"""
for h in self.handlers:
h.close()
def log(self, severity, message, *args):
+ """Send a message of a given severity to the log. If additional
+ arguments are provided, write 'message % args'. """
self._log(severity, message, args)
def _log(self, severity, message, args):
+ """Helper method: If we aren't ignoring messages of level 'severity',
+ then send message%args to all the underlying log handlers."""
+
# Enable this block to bail early in production versions
#if _SEVERITIES.get(severity, 100) < self.severity:
# return
@@ -380,18 +410,31 @@
h.write(severity, m)
def trace(self, message, *args):
+ "Write a trace (hyperverbose) message to the log"
self.log("TRACE", message, *args)
def debug(self, message, *args):
+ "Write a debug (verbose) message to the log"
self.log("DEBUG", message, *args)
def info(self, message, *args):
+ "Write an info (non-error) message to the log"
self.log("INFO", message, *args)
def warn(self, message, *args):
+ "Write a warn (recoverable error) message to the log"
self.log("WARN", message, *args)
def error(self, message, *args):
+ "Write an error (message loss error) message to the log"
self.log("ERROR", message, *args)
def fatal(self, message, *args):
+ "Write a fatal (unrecoverable system error) message to the log"
self.log("FATAL", message, *args)
def log_exc(self, severity, (exclass, ex, tb), message=None, *args):
+ """Write an exception and stack trace to the log. If message and
+ args are provided, use them as an explanitory message; otherwise,
+ introduce the message as "Unexpected exception".
+
+ This should usually be called as
+ LOG.log_exc('ERROR', sys.exc_info(), message, args...)
+ """
if message is not None:
self.log(severity, message, *args)
elif tb is not None:
@@ -408,19 +451,15 @@
self._log(severity, indented, None)
def error_exc(self, (exclass, ex, tb), message=None, *args):
+ "Same as log_exc, but logs an error message."
self.log_exc("ERROR", (exclass, ex, tb), message, *args)
def fatal_exc(self, (exclass, ex, tb), message=None, *args):
+ "Same as log_exc, but logs a fatal message."
self.log_exc("FATAL", (exclass, ex, tb), message, *args)
-_THE_LOG = None
-def getLog():
- """Return the MixMinion log object."""
- global _THE_LOG
- if _THE_LOG is None:
- _THE_LOG = Log('WARN')
-
- return _THE_LOG
+# The global 'Log' instance for the mixminion client or server.
+LOG = Log('WARN')
#----------------------------------------------------------------------
# Time processing
Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- Config.py 7 Dec 2002 04:03:35 -0000 1.21
+++ Config.py 9 Dec 2002 04:47:40 -0000 1.22
@@ -55,19 +55,24 @@
from cStringIO import StringIO
import mixminion.Common
-from mixminion.Common import MixError, getLog
+from mixminion.Common import MixError, LOG
import mixminion.Packet
import mixminion.Crypto
# String with all characters 0..255; used for str.translate
_ALLCHARS = "".join(map(chr, range(256)))
-# String with all printing ascii characters.
+# String with all printing ascii characters; used for str.translate
_GOODCHARS = "".join(map(chr, range(0x07,0x0e)+range(0x20,0x80)))
class ConfigError(MixError):
"""Thrown when an error is found in a configuration file."""
pass
+#----------------------------------------------------------------------
+# Validation functions. These are used to convert values as they appear
+# in configuration files and server descriptors into corresponding Python
+# objects, and validate their formats
+
def _parseBoolean(boolean):
"""Entry validation function. Converts a config value to a boolean.
Raises ConfigError on failure."""
@@ -111,7 +116,7 @@
'month': 60*60*24*30, # These last two aren't quite right, but we
'year': 60*60*24*365, # don't need exactness.
}
-_abbrev_units = { 'sec' : 'second', 'min': 'minute', 'mon': 'month' }
+_canonical_unit_names = { 'sec' : 'second', 'min': 'minute', 'mon' : 'month' }
def _parseInterval(interval):
"""Validation function. Converts a config value to an interval of time,
in the format (number of units, name of unit, total number of seconds).
@@ -121,9 +126,8 @@
if not m:
raise ConfigError("Unrecognized interval %r" % inter)
num, unit = float(m.group(1)), m.group(2)
- unit = _abbrev_units.get(unit, unit)
nsec = num * _seconds_per_unit[unit]
- return num, unit, nsec
+ return num, _canonical_unit_names.get(unit,unit), nsec
def _parseInt(integer):
"""Validation function. Converts a config value to an int.
@@ -131,9 +135,10 @@
i = integer.strip()
try:
return int(i)
- except ValueError, _:
+ except ValueError:
raise ConfigError("Expected an integer but got %r" % (integer))
+# Regular expression to match a dotted quad.
_ip_re = re.compile(r'\d+\.\d+\.\d+\.\d+')
def _parseIP(ip):
@@ -148,11 +153,13 @@
raise ConfigError("Invalid IP %r" % i)
try:
socket.inet_aton(i)
- except socket.error, _:
+ except socket.error:
raise ConfigError("Invalid IP %r" % i)
return i
+# Regular expression to match 'address sets' as used in Allow/Deny
+# configuration lines. General format is "<IP|*> ['/'MASK] [PORT['-'PORT]]"
_address_set_re = re.compile(r'''(\d+\.\d+\.\d+\.\d+|\*)
\s*
(?:/\s*(\d+\.\d+\.\d+\.\d+))?\s*
@@ -161,7 +168,7 @@
)?''',re.X)
def _parseAddressSet_allow(s, allowMode=1):
"""Validation function. Converts an address set string of the form
- IP/mask port-port into a tuple of (IP, Mask, Portmin, Portmax).
+ 'IP/mask port-port' into a tuple of (IP, Mask, Portmin, Portmax).
Raises ConfigError on failure."""
s = s.strip()
m = _address_set_re.match(s)
@@ -227,7 +234,7 @@
return binascii.a2b_hex(s)
else:
return binascii.a2b_base64(s)
- except (TypeError, binascii.Error, binascii.Incomplete), _:
+ except (TypeError, binascii.Error, binascii.Incomplete):
raise ConfigError("Invalid Base64 data")
def _parseHex(s):
@@ -250,11 +257,14 @@
raise ConfigError("Invalid exponent on public key")
return key
+# Regular expression to match YYYY/MM/DD
_date_re = re.compile(r"(\d\d\d\d)/(\d\d)/(\d\d)")
+# Regular expression to match YYYY/MM/DD HH:MM:SS
_time_re = re.compile(r"(\d\d\d\d)/(\d\d)/(\d\d) (\d\d):(\d\d):(\d\d)")
def _parseDate(s,_timeMode=0):
"""Validation function. Converts from YYYY/MM/DD format to a (long)
time value for midnight on that date."""
+ # If _timeMode is true, convert from YYYY/MM/DD HH:MM:SS instead.
s = s.strip()
r = (_date_re, _time_re)[_timeMode]
m = r.match(s)
@@ -284,6 +294,7 @@
_section_re = re.compile(r'\[([^\]]+)\]')
# Regular expression to match the first line of an entry
_entry_re = re.compile(r'([^:= \t]+)(?:\s*[:=]|[ \t])\s*(.*)')
+# Regular expression to match an entry from a restricted file.
_restricted_entry_re = re.compile(r'([^:= \t]+): (.*)')
def _readConfigLine(line, restrict=0):
"""Helper function. Given a line of a configuration file, return
@@ -295,6 +306,8 @@
'ENT': The line is the first line of an entry. VALUE is a (K,V) pair.
'MORE': The line is a continuation line of an entry. VALUE is the
contents of the line.
+
+ If 'restrict' is true, only accept 'ENT' lines of the format "K: V"
"""
if line == '':
@@ -302,16 +315,20 @@
space = line[0] and line[0] in ' \t'
line = line.strip()
+ # If we have an all-space line, or comment, we have no data.
if line == '' or line[0] == '#':
return None, None
- elif line[0] == '[':
+ # If the line starts with space, it's a continuation.
+ elif space:
+ return "MORE", line
+ # If the line starts with '[', we've probably got a section heading.
+ elif line[0] == '[' and not space:
m = _section_re.match(line)
if not m:
return "ERR", "Bad section declaration"
return 'SEC', m.group(1).strip()
- elif space:
- return "MORE", line
else:
+ # Now try to parse the line.
if restrict:
m = _restricted_entry_re.match(line)
else:
@@ -327,11 +344,14 @@
Throws ConfigError if the file is malformatted.
"""
+ # List of (heading, [(key, val, lineno), ...])
sections = []
+ # [(key, val, lineno)] for the current section.
curSection = None
+ # Current line number
lineno = 0
- lastKey = None
+ # Make sure all characters in the file are ASCII.
badchars = contents.translate(_ALLCHARS, _GOODCHARS)
if badchars:
raise ConfigError("Invalid characters in file: %r", badchars)
@@ -348,18 +368,20 @@
elif type == 'SEC':
curSection = [ ]
sections.append( (val, curSection) )
+ lastKey = None
elif type == 'ENT':
key,val = val
if curSection is None:
raise ConfigError("Unknown section at line %s" %lineno)
- curSection.append( [key, val, lineno] )
- lastKey = key
+ curSection.append( (key, val, lineno) )
elif type == 'MORE':
if restrict:
raise ConfigError("Continuation not allowed at line %s"%lineno)
- if not lastKey:
+ if not curSection:
raise ConfigError("Unexpected indentation at line %s" %lineno)
- curSection[-1][1] = "%s %s" % (curSection[-1][1], val)
+ lastLine = curSection[-1]
+ curSection[-1] = (lastLine[0],
+ "%s %s" % (lastLine[1], val),lastLine[2])
else:
assert type is None
if restrict:
@@ -497,7 +519,7 @@
secConfig = self._syntax.get(secName, None)
if not secConfig:
- getLog().warn("Skipping unrecognized section %s", secName)
+ LOG.warn("Skipping unrecognized section %s", secName)
continue
# Set entries from the section, searching for bad entries
@@ -645,7 +667,7 @@
if not 0 < p <= 16:
raise ConfigError("Path length must be between 1 and 16")
if p < 4:
- getLog().warn("Your default path length is frighteningly low."
+ LOG.warn("Your default path length is frighteningly low."
" I'll trust that you know what you're doing.")
SERVER_SYNTAX = {
@@ -710,7 +732,7 @@
_ConfigFile.__init__(self, fname, string)
def validate(self, sections, entries, lines, contents):
- log = getLog()
+ log = LOG
_validateHostSection(sections.get('Host', {}))
# Server section
server = sections['Server']
@@ -746,7 +768,7 @@
accordingly."""
self.moduleManager.setPath(section.get('ModulePath', None))
for mod in section.get('Module', []):
- getLog().info("Loading module %s", mod)
+ LOG.info("Loading module %s", mod)
self.moduleManager.loadExtModule(mod)
self._syntax.update(self.moduleManager.getConfigSyntax())
Index: Crypto.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Crypto.py,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- Crypto.py 2 Dec 2002 03:22:28 -0000 1.24
+++ Crypto.py 9 Dec 2002 04:47:40 -0000 1.25
@@ -15,7 +15,7 @@
from types import StringType
import mixminion._minionlib as _ml
-from mixminion.Common import MixError, MixFatalError, floorDiv, ceilDiv, getLog
+from mixminion.Common import MixError, MixFatalError, floorDiv, ceilDiv, LOG
__all__ = [ 'CryptoError', 'init_crypto', 'sha1', 'ctr_crypt', 'prng',
'strxor', 'lioness_encrypt', 'lioness_decrypt',
@@ -29,7 +29,9 @@
'APPLICATION_KEY_MODE', 'PAYLOAD_ENCRYPT_MODE',
'HIDE_HEADER_MODE' ]
+# Expose _minionlib.CryptoError as Crypto.CryptoError
CryptoError = _ml.CryptoError
+# Expose _minionlib.generate_cert
generate_cert = _ml.generate_cert
# Number of bytes in an AES key.
@@ -40,18 +42,17 @@
def init_crypto(config=None):
"""Initialize the crypto subsystem."""
configure_trng(config)
- trng(1)
try:
# Try to read /dev/urandom
trng(1)
- except MixFatalError, _:
+ except MixFatalError:
raise
except:
raise MixFatalError("Error initializing entropy source")
openssl_seed(40)
def sha1(s):
- """Return the SHA1 hash of its argument"""
+ """Return the SHA1 hash of a string"""
return _ml.sha1(s)
def strxor(s1, s2):
@@ -87,21 +88,30 @@
assert len(key2) == len(key4) == DIGEST_LEN
assert len(s) > DIGEST_LEN
+ # Split the message.
left = s[:DIGEST_LEN]
right = s[DIGEST_LEN:]
del s
# Performance note: This business with sha1("".join((key,right,key)))
- # may look slow, but it contributes only a 6% to the hashing step,
- # which in turn contributes under 11% of the time for LIONESS.
+ # may look slow, but it contributes only .7% to the total time for
+ # LIONESS.
right = _ml.aes_ctr128_crypt(
- _ml.aes_key(_ml.sha1("".join((key1,left,key1)))[:AES_KEY_LEN]),
- right, 0)
+ _ml.aes_key(_ml.sha1("".join((key1,left,key1)))[:AES_KEY_LEN]),
+ right, 0)
left = _ml.strxor(left, _ml.sha1("".join((key2,right,key2))))
right = _ml.aes_ctr128_crypt(
_ml.aes_key(_ml.sha1("".join((key3,left,key3)))[:AES_KEY_LEN]),
- right, 0)
+ right, 0)
left = _ml.strxor(left, _ml.sha1("".join((key4,right,key4))))
+ # You could write the above as:
+ # right = ctr_crypt(right, "".join((key1,left,key1))[:AES_KEY_LEN])
+ # left = strxor(left, sha1("".join((key2,right,key2))))
+ # right = ctr_crypt(right, "".join((key3,left,key3))[:AES_KEY_LEN])
+ # left = strxor(left, sha1("".join((key4,right,key4))))
+ # but that would be slower by about 10%. (Since LIONESS is in the
+ # critical path, we care.)
+
return left + right
def lioness_decrypt(s,(key1,key2,key3,key4)):
@@ -116,10 +126,23 @@
left = s[:DIGEST_LEN]
right = s[DIGEST_LEN:]
del s
- left = _ml.strxor(left, _ml.sha1("".join([key4,right,key4])))
- right = ctr_crypt(right, _ml.sha1("".join([key3,left,key3]))[:AES_KEY_LEN])
- left = _ml.strxor(left, _ml.sha1("".join([key2,right,key2])))
- right = ctr_crypt(right, _ml.sha1("".join([key1,left,key1]))[:AES_KEY_LEN])
+
+ # Slow, comprehensible version:
+ #left = strxor(left, sha1("".join([key4,right,key4])))
+ #right = ctr_crypt(right, sha1("".join([key3,left,key3]))[:AES_KEY_LEN])
+ #left = strxor(left, sha1("".join([key2,right,key2])))
+ #right = ctr_crypt(right, sha1("".join([key1,left,key1]))[:AES_KEY_LEN])
+
+ # Equivalent-but-faster version:
+ left = _ml.strxor(left, _ml.sha1("".join((key4,right,key4))))
+ right = _ml.aes_ctr128_crypt(
+ _ml.aes_key(_ml.sha1("".join((key3,left, key3)))[:AES_KEY_LEN]),
+ right, 0)
+ left = _ml.strxor(left, _ml.sha1("".join((key2,right,key2))))
+ right = _ml.aes_ctr128_crypt(
+ _ml.aes_key(_ml.sha1("".join((key1,left, key1)))[:AES_KEY_LEN]),
+ right, 0)
+
return left + right
def bear_encrypt(s,(key1,key2)):
@@ -164,7 +187,7 @@
"""
return _theTrueRNG.getBytes(count)
-# Specified in the Mixminion spec.
+# Specified in the Mixminion spec. It's a Thomas Paine quotation.
OAEP_PARAMETER = "He who would make his own liberty secure, "+\
"must guard even his enemy from oppression."
@@ -182,6 +205,7 @@
in key."""
bytes = key.get_modulus_bytes()
data = add_oaep(data,OAEP_PARAMETER,bytes)
+ # private key encrypt
return key.crypt(data, 0, 1)
def pk_decrypt(data,key):
@@ -496,8 +520,10 @@
while 1:
# Get a random positive int between 0 and 0x7fffffff.
b = self.getBytes(4)
- o = ((((((_ord(b[0])&0x7f)<<8) + _ord(b[1]))<<8) +
- _ord(b[2]))<<8) + _ord(b[3])
+ o = (((((((_ord(b[0])&0x7f)<<8) +
+ _ord(b[1]))<<8) +
+ _ord(b[2]))<<8) +
+ _ord(b[3]))
# Retry if we got a value that would fall in an incomplete
# run of 'max' elements.
if 0x7fffffff - max >= o:
@@ -519,6 +545,9 @@
class AESCounterPRNG(RNG):
'''Pseudorandom number generator that yields an AES counter-mode cipher'''
+ ## Fields:
+ # counter: the current index into the AES counter-mode keystream
+ # key: the current AES key.
def __init__(self, seed=None):
"""Creates a new AESCounterPRNG with a given seed. If no seed
is specified, gets one from the true random number generator."""
@@ -572,10 +601,13 @@
else:
requestedFile = None
+ # Build a list of candidates
defaults = PLATFORM_TRNG_DEFAULTS.get(sys.platform,
- PLATFORM_TRNG_DEFAULTS['***'])
+ PLATFORM_TRNG_DEFAULTS['***'])
files = [ requestedFile ] + defaults
+ # Now find the first of our candidates that exists and is a character
+ # device.
randFile = None
for file in files:
if file is None:
@@ -584,37 +616,27 @@
verbose = 1#(file == requestedFile)
if not os.path.exists(file):
if verbose:
- getLog().error("No such file as %s", file)
+ LOG.error("No such file as %s", file)
else:
st = os.stat(file)
if not (st[stat.ST_MODE] & stat.S_IFCHR):
if verbose:
- getLog().error("Entropy source %s isn't a character device",
+ LOG.error("Entropy source %s isn't a character device",
file)
else:
randFile = file
break
if randFile is None and _TRNG_FILENAME is None:
- getLog().fatal("No entropy source available")
+ LOG.fatal("No entropy source available")
raise MixFatalError("No entropy source available")
elif randFile is None:
- getLog().warn("Falling back to previous entropy source %s",
- _TRNG_FILENAME)
+ LOG.warn("Falling back to previous entropy source %s",
+ _TRNG_FILENAME)
else:
- getLog().info("Setting entropy source to %r", randFile)
+ LOG.info("Setting entropy source to %r", randFile)
_TRNG_FILENAME = randFile
-def _trng_uncached(n):
- '''Underlying access to our true entropy source.'''
- if _TRNG_FILENAME is None:
- configure_trng(None)
-
- f = open(_TRNG_FILENAME, 'rb')
- d = f.read(n)
- f.close()
- return d
-
class _TrueRNG(RNG):
'''Random number generator that yields pieces of entropy from
our true rng.'''
@@ -624,7 +646,14 @@
RNG.__init__(self,n)
def _prng(self,n):
"Returns n fresh bytes from our true RNG."
- return _trng_uncached(n)
+ if _TRNG_FILENAME is None:
+ configure_trng(None)
+ f = open(_TRNG_FILENAME, 'rb')
+ d = f.read(n)
+ f.close()
+ return d
+
+# Global _TrueRNG instance, for use by trng().
_theTrueRNG = _TrueRNG(1024)
Index: HashLog.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/HashLog.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- HashLog.py 7 Dec 2002 04:03:35 -0000 1.14
+++ HashLog.py 9 Dec 2002 04:47:40 -0000 1.15
@@ -3,11 +3,12 @@
"""mixminion.HashLog
- Persistant memory for the hashed secrets we've seen."""
+ Persistant memory for the hashed secrets we've seen. Used by
+ PacketHandler to prevent replay attacks."""
import os
import anydbm, dumbdbm
-from mixminion.Common import MixFatalError, getLog, createPrivateDir
+from mixminion.Common import MixFatalError, LOG, createPrivateDir
from mixminion.Packet import DIGEST_LEN
__all__ = [ 'HashLog' ]
@@ -17,7 +18,8 @@
# FFFF two-copy journaling to protect against catastrophic failure that
# FFFF underlying DB code can't handle.
-_JOURNAL_OPEN_MODE = os.O_WRONLY|os.O_CREAT|getattr(os,'O_SYNC',0)
+# flags to pass to os.open when opening the journal file.
+_JOURNAL_OPEN_FLAGS = os.O_WRONLY|os.O_CREAT|getattr(os,'O_SYNC',0)
class HashLog:
"""A HashLog is a file containing a list of message digests that we've
already processed.
@@ -46,19 +48,20 @@
# we can survive crashes between 'logHash' and 'sync'.
#
# Fields:
- # log
- # journalFileName
- # journalFile
- # journal
+ # log: an anydbm instance.
+ # journalFileName: the name of our journal file
+ # journalFile: a file object for our journal file
+ # journal: a dictionary, used to cache values currently in the
+ # journal file.
def __init__(self, filename, keyid):
"""Create a new HashLog to store data in 'filename' for the key
'keyid'."""
parent = os.path.split(filename)[0]
createPrivateDir(parent)
self.log = anydbm.open(filename, 'c')
- getLog().debug("Opening database %s for packet digests", filename)
+ LOG.debug("Opening database %s for packet digests", filename)
if isinstance(self.log, dumbdbm._Database):
- getLog().warn("Warning: logging packet digests to a flat file.")
+ LOG.warn("Warning: logging packet digests to a flat file.")
try:
if self.log["KEYID"] != keyid:
raise MixFatalError("Log KEYID does not match current KEYID")
@@ -69,13 +72,14 @@
self.journal = {}
if os.path.exists(self.journalFileName):
f = open(self.journalFileName, 'r')
+ # FFFF deal with really big journals?
j = f.read()
for i in xrange(0, len(j), DIGEST_LEN):
self.journal[j[i:i+DIGEST_LEN]] = 1
f.close()
self.journalFile = os.open(self.journalFileName,
- _JOURNAL_OPEN_MODE|os.O_APPEND, 0600)
+ _JOURNAL_OPEN_FLAGS|os.O_APPEND, 0600)
def seenHash(self, hash):
"""Return true iff 'hash' has been logged before."""
@@ -91,7 +95,6 @@
"""Insert 'hash' into the database."""
assert len(hash) == DIGEST_LEN
self.journal[hash] = 1
- #self.journalFile.write(hash)
os.write(self.journalFile, hash)
def sync(self):
@@ -102,7 +105,7 @@
self.log.sync()
os.close(self.journalFile)
self.journalFile = os.open(self.journalFileName,
- _JOURNAL_OPEN_MODE|os.O_TRUNC, 0600)
+ _JOURNAL_OPEN_FLAGS|os.O_TRUNC, 0600)
self.journal = {}
def close(self):
Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- MMTPClient.py 7 Dec 2002 04:03:35 -0000 1.11
+++ MMTPClient.py 9 Dec 2002 04:47:40 -0000 1.12
@@ -18,32 +18,46 @@
import socket
import mixminion._minionlib as _ml
from mixminion.Crypto import sha1
-from mixminion.Common import MixProtocolError, getLog
+from mixminion.Common import MixProtocolError, LOG
class BlockingClientConnection:
"""A BlockingClientConnection represents a MMTP connection to a single
server.
"""
+ ## Fields:
+ # targetIP -- the dotted-quad, IPv4 address of our server.
+ # targetPort -- the port on the server
+ # targetKeyID -- sha1 hash of the ASN1 encoding of the public key we
+ # expect the server to use.
+ # context: a TLSContext object; used to create connections.
+ # sock: a TCP socket, open to the server.
+ # tls: a TLS socket, wrapping sock.
def __init__(self, targetIP, targetPort, targetKeyID):
"""Open a new connection."""
self.targetIP = targetIP
self.targetPort = targetPort
self.targetKeyID = targetKeyID
self.context = _ml.TLSContext_new()
+ self.tls = None
+ self.sock = None
def connect(self):
"""Negotiate the handshake and protocol."""
+ # Connect to the server
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setblocking(1)
- getLog().debug("Connecting to %s:%s", self.targetIP, self.targetPort)
- self.sock.connect((self.targetIP,self.targetPort))
- getLog().debug("Handshaking with %s:%s",self.targetIP, self.targetPort)
+ LOG.debug("Connecting to %s:%s", self.targetIP, self.targetPort)
+ # Do the TLS handshaking
+ self.sock.connect((self.targetIP,self.targetPort))
+ LOG.debug("Handshaking with %s:%s",self.targetIP, self.targetPort)
self.tls = self.context.sock(self.sock.fileno())
# FFFF session resumption
self.tls.connect()
- getLog().debug("Connected.")
+ LOG.debug("Connected.")
+ # Check the public key of the server to prevent man-in-the-middle
+ # attacks.
peer_pk = self.tls.get_peer_cert_pk()
keyID = sha1(peer_pk.encode_key(public=1))
if self.targetKeyID is not None and (keyID != self.targetKeyID):
@@ -55,34 +69,41 @@
# For now, we only support 1.0, but we call it 0.1 so we can
# change our mind between now and a release candidate, and so we
# can obsolete betas come release time.
- getLog().debug("Negotiatiating MMTP protocol")
+ LOG.debug("Negotiatiating MMTP protocol")
self.tls.write("MMTP 0.1\r\n")
inp = self.tls.read(len("MMTP 0.1\r\n"))
if inp != "MMTP 0.1\r\n":
raise MixProtocolError("Protocol negotiation failed")
- getLog().debug("MMTP protocol negotated: version 0.1")
+ LOG.debug("MMTP protocol negotated: version 0.1")
def sendPacket(self, packet):
"""Send a single packet to a server."""
assert len(packet) == 1<<15
- getLog().debug("Sending packet")
+ LOG.debug("Sending packet")
+ ##
+ # We write: "SEND\r\n", 28KB of data, and sha1(packet|"SEND").
self.tls.write("SEND\r\n")
self.tls.write(packet)
self.tls.write(sha1(packet+"SEND"))
- getLog().debug("Packet sent; waiting for ACK")
+ LOG.debug("Packet sent; waiting for ACK")
+ # And we expect, "RECEIVED\r\n", and sha1(packet|"RECEIVED")
inp = self.tls.read(len("RECEIVED\r\n")+20)
if inp != "RECEIVED\r\n"+sha1(packet+"RECEIVED"):
raise MixProtocolError("Bad ACK received")
- getLog().debug("ACK received; packet successfully delivered")
+ LOG.debug("ACK received; packet successfully delivered")
+
+ # FFFF we need a sendJunkPacket method.
def shutdown(self):
"""Close this connection."""
- getLog().debug("Shutting down connection to %s:%s",
+ LOG.debug("Shutting down connection to %s:%s",
self.targetIP, self.targetPort)
- self.tls.shutdown()
- self.sock.close()
- getLog().debug("Connection closed")
+ if self.tls is not None:
+ self.tls.shutdown()
+ if self.sock is not None:
+ self.sock.close()
+ LOG.debug("Connection closed")
def sendMessages(targetIP, targetPort, targetKeyID, packetList):
"""Sends a list of messages to a server."""
Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- MMTPServer.py 7 Dec 2002 04:03:35 -0000 1.19
+++ MMTPServer.py 9 Dec 2002 04:47:40 -0000 1.20
@@ -29,18 +29,18 @@
from types import StringType
import mixminion._minionlib as _ml
-from mixminion.Common import MixError, MixFatalError, getLog, stringContains
+from mixminion.Common import MixError, MixFatalError, LOG, stringContains
from mixminion.Crypto import sha1
from mixminion.Packet import MESSAGE_LEN, DIGEST_LEN
__all__ = [ 'AsyncServer', 'ListenConnection', 'MMTPServerConnection',
'MMTPClientConnection' ]
-trace = getLog().trace
-info = getLog().info
-debug = getLog().info
-warn = getLog().warn
-error = getLog().error
+trace = LOG.trace
+info = LOG.info
+debug = LOG.info
+warn = LOG.warn
+error = LOG.error
class AsyncServer:
"""AsyncServer is the core of a general-purpose asynchronous
@@ -49,6 +49,11 @@
(respectively), and waits for their underlying sockets to be
available for the desired operations.
"""
+ ## Fields:
+ # writers: map from fd to 'Connection' objects that are interested
+ # in write events.
+ # readers: map from fd to 'Connection' objects that are interested
+ # in read events.
def __init__(self):
"""Create a new AsyncServer with no readers or writers."""
self.writers = {}
@@ -152,6 +157,13 @@
"""A ListenConnection listens on a given port/ip combination, and calls
a 'connectionFactory' method whenever a new connection is made to that
port."""
+ ## Fields:
+ # ip: IP to listen on.
+ # port: port to listen on.
+ # sock: socket to bind.
+ # connectionFactory: a function that takes as input a socket from a
+ # newly received connection, and returns a Connection object to
+ # register with the async server.
def __init__(self, ip, port, backlog, connectionFactory):
"""Create a new ListenConnection"""
self.ip = ip
@@ -207,7 +219,8 @@
#
# __con: an underlying TLS object
# __state: a callback to use whenever we get a read or a write. May
- # throw _ml.TLSWantRead or _ml.TLSWantWrite.
+ # throw _ml.TLSWantRead or _ml.TLSWantWrite. See __acceptFn,
+ # __connectFn, __shutdownFn, __readFn, __writeFn.
# __server: an AsyncServer.
# __inbuf: A list of strings that we've read since the last expectRead.
# __inbuflen: The total length of all the strings in __inbuf
@@ -216,7 +229,6 @@
# __terminator: None, or a string which will terminate the current read.
# __outbuf: None, or the remainder of the string we're currently
# writing.
-
def __init__(self, sock, tls, serverMode):
"""Create a new SimpleTLSConnection.
@@ -430,10 +442,17 @@
pass
#----------------------------------------------------------------------
+# Implementation for MMTP.
+
+# The protocol string to send.
PROTOCOL_STRING = "MMTP 0.1\r\n"
-PROTOCOL_RE = re.compile("MMTP ([^\s\r\n]+)\r\n")
+# The protocol specification to expect.
+PROTOCOL_RE = re.compile("MMTP ([^\s\r\n]+)\r\n")
+# Control line for sending a message.
SEND_CONTROL = "SEND\r\n"
+# Control line for sending padding.
JUNK_CONTROL = "JUNK\r\n"
+# Control line for acknowledging a message
RECEIVED_CONTROL = "RECEIVED\r\n"
SEND_CONTROL_LEN = len(SEND_CONTROL)
RECEIVED_CONTROL_LEN = len(RECEIVED_CONTROL)
@@ -443,7 +462,14 @@
class MMTPServerConnection(SimpleTLSConnection):
'''An asynchronous implementation of the receiving side of an MMTP
connection.'''
+ ## Fields:
+ # messageConsumer: a function to call with all received messages.
+ # finished: callback when we're done with a read or write; see
+ # SimpleTLSConnection.
def __init__(self, sock, tls, consumer):
+ """Create an MMTP connection to receive messages sent along a given
+ socket. When valid packets are received, pass them to the
+ function 'consumer'."""
SimpleTLSConnection.__init__(self, sock, tls, 1)
self.messageConsumer = consumer
self.finished = self.__setupFinished
@@ -528,6 +554,9 @@
class MMTPClientConnection(SimpleTLSConnection):
"""Asynchronious implementation of the sending ("client") side of a
mixminion connection."""
+ ## Fields:
+ # ip, port, keyID, messageList, handleList, sendCallback, failCallback:
+ # As described in the docstring for __init__ below.
def __init__(self, context, ip, port, keyID, messageList, handleList,
sentCallback=None, failCallback=None):
"""Create a connection to send messages to an MMTP server.
@@ -656,9 +685,10 @@
self.failCallback(msg,handle,retriable)
LISTEN_BACKLOG = 10 # ???? Is something else more reasonable?
-class MMTPServer(AsyncServer):
+class MMTPAsyncServer(AsyncServer):
"""A helper class to invoke AsyncServer, MMTPServerConnection, and
- MMTPClientConnection"""
+ MMTPClientConnection, with a function to add new connections, and
+ callbacks for message success and failure."""
def __init__(self, config, tls):
AsyncServer.__init__(self)
@@ -686,7 +716,7 @@
self.listener.shutdown()
def sendMessages(self, ip, port, keyID, messages, handles):
- """Send a set of messages to a given server."""
+ """Begin sending a set of messages to a given server."""
# ???? Can we remove these asserts yet?
for m,h in zip(messages, handles):
assert len(m) == MESSAGE_LEN
@@ -694,7 +724,8 @@
con = MMTPClientConnection(self.context,
ip, port, keyID, messages, handles,
- self.onMessageSent, self.onMessageUndeliverable)
+ self.onMessageSent,
+ self.onMessageUndeliverable)
con.register(self)
def onMessageReceived(self, msg):
Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- Main.py 2 Dec 2002 03:47:46 -0000 1.10
+++ Main.py 9 Dec 2002 04:47:40 -0000 1.11
@@ -16,8 +16,6 @@
# print>>, no automatic string concatenation and no import foo.bar.
import sys
-import stat
-import os
# Check: are we running a version earlier than 2.0? If so, die.
if not hasattr(sys,'version_info') or sys.version_info[0] < 2:
@@ -28,7 +26,10 @@
" You seem to be running version %s.\n")%_ver)
sys.exit(1)
+import os
+import stat
import getopt
+import types
def filesAreSame(f1, f2):
"Return true if f1 and f2 are exactly the same file."
@@ -39,7 +40,7 @@
ino1 = os.stat(f1)[stat.ST_INO]
ino2 = os.stat(f2)[stat.ST_INO]
return ino1 and ino1 > 0 and ino1 == ino2
- except OSError, _:
+ except OSError:
return 0
def correctPath(myself):
@@ -48,6 +49,14 @@
# wind up somewhere appropriate on pythonpath. This isn't good enough,
# however: we want to run even when sysadmins don't understand distutils.)
+ # If we can import mixminion.Main, we bail out early: let's not mess
+ # with anything.
+ try:
+ __import__('mixminion.Main')
+ return
+ except ImportError:
+ pass
+
orig_cmd = myself
# First, resolve all links.
while os.path.islink(myself):
@@ -67,6 +76,10 @@
parentdir = os.path.normpath(parentdir)
foundEntry = 0
for pathEntry in sys.path:
+ # There are intimations on Python-dev that sys.path may eventually
+ # contain non-strings.
+ if not isinstance(pathEntry, types.StringType):
+ continue
if os.path.normpath(pathEntry) == parentdir:
foundEntry = 1; break
@@ -89,11 +102,11 @@
sys.exit(1)
# Global map from command name to 2-tuples of (module_name, function_name).
+# The function 'main' below uses this map to pick which module to import,
+# and which function to invoke.
#
-# DOCDOC unclear!
-#
-# 'Main.py <cmd> arg1 arg2 arg3' will result in a call to function_name
-# in module_name. The function should take two arguments: a string to
+# 'Main.py <cmd> arg1 arg2 arg3' will result in a call to <function_name>
+# in <module_name>. The function should take two arguments: a string to
# be used as command name in error messages, and a list of [arg1,arg2,arg3].'
#
# By convention, all commands must print a usage message and exit when
Index: Modules.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Modules.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- Modules.py 7 Dec 2002 04:03:35 -0000 1.22
+++ Modules.py 9 Dec 2002 04:47:40 -0000 1.23
@@ -3,7 +3,10 @@
"""mixminion.Modules
- Type codes and dispatch functions for routing functionality."""
+ Code to support pluggable exit module functionality; implementation
+ for built-in modules.
+ """
+# FFFF We may, someday, want to support non-exit modules.
__all__ = [ 'ModuleManager', 'DeliveryModule',
'DROP_TYPE', 'FWD_TYPE', 'SWAP_FWD_TYPE',
@@ -22,7 +25,7 @@
import mixminion.Queue
import mixminion.BuildMessage
from mixminion.Config import ConfigError, _parseBoolean, _parseCommand
-from mixminion.Common import getLog, createPrivateDir, MixError
+from mixminion.Common import LOG, createPrivateDir, MixError
# Return values for processMessage
DELIVER_OK = 1
@@ -32,6 +35,7 @@
# Numerically first exit type.
MIN_EXIT_TYPE = 0x0100
+# XXXX001 move these into Packet.py ===================================START
# Mixminion types
DROP_TYPE = 0x0000 # Drop the current message
FWD_TYPE = 0x0001 # Forward the msg to an IPV4 addr via MMTP
@@ -40,6 +44,7 @@
# Exit types
SMTP_TYPE = 0x0100 # Mail the message
MBOX_TYPE = 0x0101 # Send the message to one of a fixed list of addresses
+# XXXX001 move these into Packet.py =====================================END
class DeliveryModule:
"""Abstract base for modules; delivery modules should implement
@@ -53,6 +58,8 @@
* It must know its own name.
* It must know which types it handles.
* Of course, it needs to know how to deliver a message."""
+ # FFFF DeliveryModules need to know about the AsyncServer object in
+ # FFFF case they support asynchronous delivery.
def __init__(self):
"Zero-argument constructor, as required by Module protocol."
pass
@@ -105,28 +112,34 @@
DELIVER_OK (if the message was successfully delivered),
DELIVER_FAIL_RETRY (if the message wasn't delivered, but might be
deliverable later), or
- DELIVER_FAIL_NORETRY (if the message shouldn't be tried later)."""
- raise NotImplementedError("processMessage")
+ DELIVER_FAIL_NORETRY (if the message shouldn't be tried later).
+ (This method is only used by your delivery queue; if you use
+ a nonstandard delivery queue, you don't need to implement this."""
+ raise NotImplementedError("processMessage")
class ImmediateDeliveryQueue:
"""Helper class usable as delivery queue for modules that don't
actually want a queue. Such modules should have very speedy
processMessage() methods, and should never have deliery fail."""
+ ##Fields:
+ # module: the underlying DeliveryModule object.
def __init__(self, module):
self.module = module
def queueDeliveryMessage(self, (exitType, address, tag), message):
+ """Instead of queueing our message, pass it directly to the underlying
+ DeliveryModule."""
try:
res = self.module.processMessage(message, tag, exitType, address)
if res == DELIVER_OK:
return
elif res == DELIVER_FAIL_RETRY:
- getLog().error("Unable to retry delivery for message")
+ LOG.error("Unable to retry delivery for message")
else:
- getLog().error("Unable to deliver message")
+ LOG.error("Unable to deliver message")
except:
- getLog().error_exc(sys.exc_info(),
+ LOG.error_exc(sys.exc_info(),
"Exception delivering message")
def sendReadyMessages(self):
@@ -136,6 +149,8 @@
class SimpleModuleDeliveryQueue(mixminion.Queue.DeliveryQueue):
"""Helper class used as a default delivery queue for modules that
don't care about batching messages to like addresses."""
+ ## Fields:
+ # module: the underlying module.
def __init__(self, module, directory):
mixminion.Queue.DeliveryQueue.__init__(self, directory)
self.module = module
@@ -150,10 +165,10 @@
elif result == DELIVER_FAIL_RETRY:
self.deliveryFailed(handle, 1)
else:
- getLog().error("Unable to deliver message")
+ LOG.error("Unable to deliver message")
self.deliveryFailed(handle, 0)
except:
- getLog().error_exc(sys.exc_info(),
+ LOG.error_exc(sys.exc_info(),
"Exception delivering message")
self.deliveryFailed(handle, 0)
@@ -166,9 +181,13 @@
the ModuleManager, but will not receive messags until it is
enabled.
- Because modules need to tell the ServerConfig object aboutt their
+ Because modules need to tell the ServerConfig object about their
configuration options, initializing the ModuleManager is usually done
- through ServerConfig. See ServerConfig.getModuleManager()."""
+ through ServerConfig. See ServerConfig.getModuleManager().
+
+ To send messages, call 'queueMessage' for each message to send, then
+ call 'sendReadyMessages'.
+ """
##
# Fields
# syntax: extensions to the syntax configuration in Config.py
@@ -210,7 +229,7 @@
def registerModule(self, module):
"""Inform this ModuleManager about a delivery module. This method
updates the syntax options, but does not enable the module."""
- getLog().info("Loading module %s", module.getName())
+ LOG.info("Loading module %s", module.getName())
self.modules.append(module)
syn = module.getConfigSyntax()
for sec, rules in syn.items():
@@ -234,7 +253,7 @@
pyPkg = ".".join(ids[:-1])
pyClassName = ids[-1]
orig_path = sys.path[:]
- getLog().info("Loading module %s", className)
+ LOG.info("Loading module %s", className)
try:
sys.path[0:0] = self.path
try:
@@ -253,6 +272,7 @@
raise MixError("Error initializing module %s" %className)
def validate(self, sections, entries, lines, contents):
+ # (As in ServerConfig)
for m in self.modules:
m.validateConfig(sections, entries, lines, contents)
@@ -269,10 +289,10 @@
for t in module.getExitTypes():
if (self.typeToModule.has_key(t) and
self.typeToModule[t].getName() != module.getName()):
- getLog().warn("More than one module is enabled for type %x"%t)
+ LOG.warn("More than one module is enabled for type %x"%t)
self.typeToModule[t] = module
- getLog().info("Module %s: enabled for types %s",
+ LOG.info("Module %s: enabled for types %s",
module.getName(),
map(hex, module.getExitTypes()))
@@ -282,12 +302,13 @@
self.enabled[module.getName()] = 1
def cleanQueues(self):
+ """Remove trash messages from all internal queues."""
for queue in self.queues.values():
queue.cleanQueue()
def disableModule(self, module):
"""Unmaps all the types for a module object."""
- getLog().info("Disabling module %s", module.getName())
+ LOG.info("Disabling module %s", module.getName())
for t in module.getExitTypes():
if (self.typeToModule.has_key(t) and
self.typeToModule[t].getName() == module.getName()):
@@ -298,17 +319,19 @@
del self.enabled[module.getName()]
def queueMessage(self, message, tag, exitType, address):
+ """Queue a message for delivery."""
+ # FFFF Support non-exit messages.
mod = self.typeToModule.get(exitType, None)
if mod is None:
- getLog().error("Unable to handle message with unknown type %s",
+ LOG.error("Unable to handle message with unknown type %s",
exitType)
return
queue = self.queues[mod.getName()]
- getLog().debug("Delivering message %r (type %04x) via module %s",
+ LOG.debug("Delivering message %r (type %04x) via module %s",
message[:8], exitType, mod.getName())
try:
payload = mixminion.BuildMessage.decodePayload(message, tag)
- except MixError, _:
+ except MixError:
queue.queueDeliveryMessage((exitType, address, 'err'), message)
return
if payload is None:
@@ -342,19 +365,35 @@
def createDeliveryQueue(self, directory):
return ImmediateDeliveryQueue(self)
def processMessage(self, message, tag, exitType, exitInfo):
- getLog().debug("Dropping padding message")
+ LOG.debug("Dropping padding message")
return DELIVER_OK
#----------------------------------------------------------------------
class MBoxModule(DeliveryModule):
- # FFFF This implementation can stall badly if we don't have a fast
- # FFFF local MTA.
+ """Implementation for MBOX delivery: sends messages, via SMTP, to
+ addresses from a local file. The file must have the format
+ addr: smtpaddr
+ addr: smtpaddr
+ ...
+
+ When we receive a message send to 'addr', we deliver it to smtpaddr.
+ """
+ ##
+ # Fields:
+ # addresses: a map from address to SMTP address
+ # server: the name of our SMTP server
+ # addressFile: the location of our address file
+ # returnAddress: the address we use in our 'From' line
+ # contact: the contact address we mention in our boilerplate
+ # nickname: our server nickname; for use in our boilerplate
+ # addr: our IP address, or "<Unknown IP>": for use in our boilerplate.
def __init__(self):
DeliveryModule.__init__(self)
- self.enabled = 0
self.addresses = {}
def getConfigSyntax(self):
+ # FFFF There should be some way to say that fields are required
+ # FFFF if the module is enabled.
return { "Delivery/MBOX" :
{ 'Enabled' : ('REQUIRE', _parseBoolean, "no"),
'AddressFile' : ('ALLOW', None, None),
@@ -369,9 +408,7 @@
def configure(self, config, moduleManager):
# XXXX001 Check this. Conside error handling
-
- self.enabled = config['Delivery/MBOX'].get("Enabled", 0)
- if not self.enabled:
+ if not config['Delivery/MBOX'].get("Enabled", 0):
moduleManager.disableModule(self)
return
@@ -389,8 +426,9 @@
self.nickname = config['Server']['Nickname']
if not self.nickname:
self.nickname = socket.gethostname()
- self.addr = config['Incoming/MMTP'].get('IP', "<Unknown host>")
+ self.addr = config['Incoming/MMTP'].get('IP', "<Unknown IP>")
+ # Parse the address file.
self.addresses = {}
f = open(self.addressFile)
address_line_re = re.compile(r'\s*([^\s:=]+)\s*[:=]\s*(\S+)')
@@ -409,7 +447,7 @@
raise ConfigError("Bad address on line %s of %s"%(
lineno,self.addressFile))
self.addresses[m.group(1)] = m.group(2)
- getLog().trace("Mapping MBOX address %s -> %s", m.group(1),
+ LOG.trace("Mapping MBOX address %s -> %s", m.group(1),
m.group(2))
finally:
f.close()
@@ -429,17 +467,20 @@
return [ MBOX_TYPE ]
def processMessage(self, message, tag, exitType, address):
+ # Determine that message's address;
assert exitType == MBOX_TYPE
- getLog().trace("Received MBOX message")
+ LOG.trace("Received MBOX message")
info = mixminion.Packet.parseMBOXInfo(address)
try:
address = self.addresses[info.user]
- except KeyError, _:
- getLog().error("Unknown MBOX user %r", info.user)
+ except KeyError:
+ LOG.error("Unknown MBOX user %r", info.user)
return DELIVER_FAIL_NORETRY
+ # Escape the message if it isn't plaintext ascii
msg = _escapeMessageForEmail(message, tag)
+ # Generate the boilerplate (FFFF Make this configurable)
fields = { 'user': address,
'return': self.returnAddress,
'nickname': self.nickname,
@@ -458,20 +499,17 @@
%(msg)s""" % fields
+ # Deliver the message
return sendSMTPMessage(self.server, [address], self.returnAddress, msg)
#----------------------------------------------------------------------
class SMTPModule(DeliveryModule):
- """Placeholder for real exit node implementation.
- DOCDOC document me."""
+ """Placeholder for real exit node implementation.
+ For now, use MixmasterSMTPModule"""
def __init__(self):
DeliveryModule.__init__(self)
- self.enabled = 0
def getServerInfoBlock(self):
- if self.enabled:
- return "[Delivery/SMTP]\nVersion: 0.1\n"
- else:
- return ""
+ return "[Delivery/SMTP]\nVersion: 0.1\n"
def getName(self):
return "SMTP"
def getExitTypes(self):
@@ -485,9 +523,18 @@
# FFFF ... or maybe we should deliberately ignore them, since
# FFFF this is only a temporary workaround until enough people
# FFFF are running SMTP exit nodes
+ ## Fields:
+ # server: The path (usually a single server) to use for outgoing messages.
+ # Multiple servers should be separated by commas.
+ # subject: The subject line we use for outgoing messages
+ # command: The Mixmaster binary.
+ # options: Options to pass to the Mixmaster binary when queueing messages
+ # tmpQueue: An auxiliary Queue used to hold files so we can pass them to
+ # Mixmaster. (This should go away; we should use stdin instead.)
+
def __init__(self):
SMTPModule.__init__(self)
- self.mixDir = None
+
def getConfigSyntax(self):
return { "Delivery/SMTP-Via-Mixmaster" :
{ 'Enabled' : ('REQUIRE', _parseBoolean, "no"),
@@ -503,8 +550,7 @@
pass
def configure(self, config, manager):
sec = config['Delivery/SMTP-Via-Mixmaster']
- self.enabled = sec.get("Enabled", 0)
- if not self.enabled:
+ if not sec.get("Enabled", 0):
manager.disableModule(self)
return
cmd = sec['MixCommand']
@@ -517,11 +563,16 @@
def getName(self):
return "SMTP_MIX2"
+
def createDeliveryQueue(self, queueDir):
+ # We create a temporary queue so we can hold files there for a little
+ # while before passing their names to mixmaster.
self.tmpQueue = mixminion.Queue.Queue(queueDir+"_tmp", 1, 1)
self.tmpQueue.removeAll()
return _MixmasterSMTPModuleDeliveryQueue(self, queueDir)
+
def processMessage(self, message, tag, exitType, smtpAddress):
+ """Insert a message into the Mixmaster queue"""
assert exitType == SMTP_TYPE
# parseSMTPInfo will raise a parse error if the mailbox is invalid.
info = mixminion.Packet.parseSMTPInfo(smtpAddress)
@@ -533,33 +584,40 @@
opts = self.options + ("-t", info.email,
self.tmpQueue.getMessagePath(handle))
code = os.spawnl(os.P_WAIT, cmd, cmd, *opts)
- getLog().debug("Queued Mixmaster message: exit code %s", code)
+ LOG.debug("Queued Mixmaster message: exit code %s", code)
self.tmpQueue.removeMessage(handle)
return DELIVER_OK
def flushMixmasterPool(self):
- "DOCDOC"
+ """Send all pending messages from the Mixmaster queue. This
+ should be called after invocations of processMessage."""
cmd = self.command
- getLog().debug("Flushing Mixmaster pool")
+ LOG.debug("Flushing Mixmaster pool")
os.spawnl(os.P_NOWAIT, cmd, cmd, "-S")
class _MixmasterSMTPModuleDeliveryQueue(SimpleModuleDeliveryQueue):
- "DOCDOC"
- def __init__(self, module, directory):
- SimpleModuleDeliveryQueue.__init__(self, module, directory)
+ """Delivery queue for _MixmasterSMTPModule. Same as
+ SimpleModuleDeliveryQueue, except that we must call flushMixmasterPool
+ after queueing messages for Mixmaster."""
def _deliverMessages(self, msgList):
SimpleModuleDeliveryQueue._deliverMessages(self, msgList)
self.module.flushMixmasterPool()
#----------------------------------------------------------------------
+
def sendSMTPMessage(server, toList, fromAddr, message):
- getLog().trace("Sending message via SMTP host %s to %s", server, toList)
+ """Send a single SMTP message. The message will be delivered to
+ toList, and seem to originate from fromAddr. We use 'server' as an
+ MTA."""
+ # FFFF This implementation can stall badly if we don't have a fast
+ # FFFF local MTA.
+ LOG.trace("Sending message via SMTP host %s to %s", server, toList)
con = smtplib.SMTP(server)
try:
con.sendmail(fromAddr, toList, message)
res = DELIVER_OK
except smtplib.SMTPException, e:
- getLog().warn("Unsuccessful smtp: "+str(e))
+ LOG.warn("Unsuccessful smtp: "+str(e))
res = DELIVER_FAIL_RETRY
con.quit()
@@ -569,19 +627,30 @@
#----------------------------------------------------------------------
+# XXXX001 There's another function like this in config.
# DOCDOC
_allChars = "".join(map(chr, range(256)))
# DOCDOC
# ????001 Are there any nonprinting chars >= 0x7f to worry about now?
_nonprinting = "".join(map(chr, range(0x00, 0x07)+range(0x0E, 0x20)))
+
def isPrintable(s):
"""Return true iff s consists only of printable characters."""
printable = s.translate(_allChars, _nonprinting)
return len(printable) == len(s)
def _escapeMessageForEmail(msg, tag):
- """DOCDOC
- -> None | str """
+ """Helper function: Given a message and tag, escape the message if
+ it is not plaintext ascii, and wrap it in some standard
+ boilerplate. Add a disclaimer if the message is not ascii.
+
+ msg -- A (possibly decoded) message
+ tag -- One of: a 20-byte decoding tag [if the message is encrypted
+ or a reply]
+ None [if the message is in plaintext]
+ 'err' [if the message was invalid.]
+
+ Returns None on an invalid message."""
m = _escapeMessage(msg, tag, text=1)
if m is None:
return None
@@ -604,16 +673,27 @@
%s%s============ ANONYMOUS MESSAGE ENDS\n""" %(junk_msg, tag, msg)
def _escapeMessage(message, tag, text=0):
- """DOCDOC
- (message,tag|None,output-as-text?)
- -> ("TXT"|"BIN"|"ENC", message, tag|None) or None
+ """Helper: given a decoded message (and possibly its tag), determine
+ whether the message is a text plaintext message (code='TXT'), a
+ binary plaintext message (code 'BIN'), or an encrypted message/reply
+ (code='ENC'). If requested, non-TXT messages are base-64 encoded.
+
+ Returns: (code, message, tag (for ENC) or None (for BIN, TXT).
+ Returns None if the message is invalid.
+
+ message -- A (possibly decoded) message
+ tag -- One of: a 20-byte decoding tag [if the message is encrypted
+ or a reply]
+ None [if the message is in plaintext]
+ 'err' [if the message was invalid.]
+ text -- flag: if true, non-TXT messages must be base64-encoded.
"""
if tag == 'err':
return None
elif tag is not None:
code = "ENC"
else:
- tag = None
+ assert tag is None
if isPrintable(message):
code = "TXT"
else:
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- Packet.py 7 Dec 2002 04:03:35 -0000 1.18
+++ Packet.py 9 Dec 2002 04:47:40 -0000 1.19
@@ -3,7 +3,11 @@
"""mixminion.Packet
Functions, classes, and constants to parse and unparse Mixminion
- messages and related structures."""
+ messages and related structures.
+
+ For functions that handle client-side generation and decoding of
+ packets, see BuildMessage.py. For functions that handle
+ server-side processing of packets, see PacketHandler.py."""
__all__ = [ 'ParseError', 'Message', 'Header', 'Subheader',
'parseMessage', 'parseHeader', 'parseSubheader',
@@ -185,12 +189,16 @@
"routinglen=%(routinglen)r)")% self.__dict__
def getExitAddress(self):
+ """Return the part of the routingInfo that contains the delivery
+ address. (Requires that routingType is an exit type.)"""
# XXXX001 SPEC This is not explicit in the spec.
assert self.routingtype >= mixminion.Modules.MIN_EXIT_TYPE
assert len(self.routinginfo) >= TAG_LEN
return self.routinginfo[TAG_LEN:]
def getTag(self):
+ """Return the part of the routingInfo that contains the decoding
+ tag. (Requires that routingType is an exit type.)"""
# XXXX001 SPEC This is not explicit in the spec.
assert self.routingtype >= mixminion.Modules.MIN_EXIT_TYPE
assert len(self.routinginfo) >= TAG_LEN
@@ -334,7 +342,11 @@
return "%s%s" % (header, self.data)
class FragmentPayload(_Payload):
- """Represents the fields of a decoded fragment payload."""
+ """Represents the fields of a decoded fragment payload.
+
+ FFFF Fragments are not yet fully supported; there's no code to generate
+ or decode them.
+ """
def __init__(self, index, hash, msgID, msgLen, data):
self.index = index
self.hash = hash
@@ -362,6 +374,11 @@
#----------------------------------------------------------------------
# REPLY BLOCKS
+# A reply block is: the string "SURB", a major number, a minor number,
+# a 4-byte "valid-until" timestamp, a 2K header, 2 bytes of routingLen for
+# the last server in the first leg; 2 bytes of routingType for the last
+# server in the first leg; a 16-byte shared end-to-end key, and the
+# routingInfo for the last server.
RB_UNPACK_PATTERN = "!4sBBL%dsHH%ss" % (HEADER_LEN, SECRET_LEN)
MIN_RB_LEN = 30+HEADER_LEN
@@ -369,7 +386,6 @@
"""Return a new ReplyBlock object for an encoded reply block"""
if len(s) < MIN_RB_LEN:
raise ParseError("Reply block too short")
-
try:
magic, major, minor, timestamp, header, rlen, rt, key = \
struct.unpack(RB_UNPACK_PATTERN, s[:MIN_RB_LEN])
@@ -453,10 +469,17 @@
return (type(self) == type(other) and self.ip == other.ip and
self.port == other.port and self.keyinfo == other.keyinfo)
-#DOCDOC
-# FFFF Support subdomains and quoted strings
+# Regular expressions to valide RFC822 addresses.
+# (This is more strict than RFC822, actually. RFC822 allows tricky
+# stuff to quote special characters, and I don't trust every MTA or
+# delivery command to support addresses like <bob@bob."; rm -rf /; echo".com>)
+
+# An 'Atom' is a non-escape, non-null, non-space, non-punctuation character.
_ATOM_PAT = r'[^\x00-\x20()\[\]()<>@,;:\\".\x7f-\xff]+'
+# The 'Local part' (and, for us, the domain portion too) is a sequence of
+# dot-separated atoms.
_LOCAL_PART_PAT = r"(?:%s)(?:\.(?:%s))*" % (_ATOM_PAT, _ATOM_PAT)
+# A mailbox is two 'local parts' separated by an @ sign.
_RFC822_PAT = r"\A%s@%s\Z" % (_LOCAL_PART_PAT, _LOCAL_PART_PAT)
RFC822_RE = re.compile(_RFC822_PAT)
Index: PacketHandler.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/PacketHandler.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- PacketHandler.py 2 Dec 2002 03:24:46 -0000 1.10
+++ PacketHandler.py 9 Dec 2002 04:47:40 -0000 1.11
@@ -1,7 +1,7 @@
# Copyright 2002 Nick Mathewson. See LICENSE for licensing information.
# $Id$
-"""mixminion.PacketHandler: Code to process mixminion packets"""
+"""mixminion.PacketHandler: Code to process mixminion packets on a server"""
import mixminion.Crypto as Crypto
import mixminion.Packet as Packet
@@ -20,7 +20,9 @@
checks, swaps headers if necessary, re-pads, and decides whether
to drop the message, relay the message, or send the message to
an exit handler."""
-
+ ## Fields:
+ # privatekey: list of RSA private keys that we accept
+ # hashlog: list of HashLog objects corresponding to the keys.
def __init__(self, privatekey, hashlog):
"""Constructs a new packet handler, given a private key object for
header encryption, and a hashlog object to prevent replays.
@@ -37,8 +39,8 @@
self.privatekey = privatekey
self.hashlog = hashlog
- except TypeError, _:
- # Privatekey must not be subscriptable; we have only one.
+ except TypeError:
+ # Privatekey is not be subscriptable; we must have only one.
self.privatekey = (privatekey, )
self.hashlog = (hashlog, )
Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- Queue.py 7 Dec 2002 04:03:35 -0000 1.22
+++ Queue.py 9 Dec 2002 04:47:40 -0000 1.23
@@ -3,7 +3,7 @@
"""mixminion.Queue
- Facility for a fairly secure, directory-based, unordered queue.
+ Facility for fairly secure, directory-based, unordered queues.
"""
import os
@@ -12,7 +12,7 @@
import stat
import cPickle
-from mixminion.Common import MixError, MixFatalError, secureDelete, getLog, \
+from mixminion.Common import MixError, MixFatalError, secureDelete, LOG, \
createPrivateDir
from mixminion.Crypto import AESCounterPRNG
@@ -21,9 +21,9 @@
# Mode to pass to open(2) for creating a new file, and dying if it already
# exists.
-_NEW_MESSAGE_MODE = os.O_WRONLY+os.O_CREAT+os.O_EXCL
+_NEW_MESSAGE_FLAGS = os.O_WRONLY+os.O_CREAT+os.O_EXCL
# On windows or mac, binary != text.
-_NEW_MESSAGE_MODE += getattr(os, 'O_BINARY', 0)
+_NEW_MESSAGE_FLAGS += getattr(os, 'O_BINARY', 0)
# Any inp_* files older than INPUT_TIMEOUT seconds old are assumed to be
# trash.
@@ -34,14 +34,13 @@
CLEAN_TIMEOUT = 120
class Queue:
- """A Queue is an unordered collection of files with secure remove and
- move operations.
+ """A Queue is an unordered collection of files with secure insert, move,
+ and delete operations.
Implementation: a queue is a directory of 'messages'. Each
filename in the directory has a name in one of the following
formats:
-
- rmv_HANDLE (A message waiting to be deleted)
+ rmv_HANDLE (A message waiting to be deleted)
msg_HANDLE (A message waiting in the queue.
inp_HANDLE (An incomplete message being created.)
(Where HANDLE is a randomly chosen 12-character selection from the
@@ -73,7 +72,7 @@
self.dir = location
if not os.path.isabs(location):
- getLog().warn("Queue path %s isn't absolute.", location)
+ LOG.warn("Queue path %s isn't absolute.", location)
if os.path.exists(location) and not os.path.isdir(location):
raise MixFatalError("%s is not a directory" % location)
@@ -184,7 +183,7 @@
commit your changes, or abortMessage to reject them."""
handle = self.__newHandle()
fname = os.path.join(self.dir, "inp_"+handle)
- fd = os.open(fname, _NEW_MESSAGE_MODE, 0600)
+ fd = os.open(fname, _NEW_MESSAGE_FLAGS, 0600)
return os.fdopen(fd, 'wb'), handle
def finishMessage(self, f, handle):
@@ -367,6 +366,8 @@
as requested, according to a mixing algorithm that sends a batch
of messages every N seconds."""
# FFFF : interval is unused.
+ ## Fields:
+ # interval: scanning interval, in seconds.
def __init__(self, location, interval=600):
"""Create a TimedMixQueue that sends its entire batch of messages
every 'interval' seconds."""
@@ -386,6 +387,12 @@
as requested, according the Cottrell (timed dynamic-pool) mixing
algorithm from Mixmaster."""
# FFFF : interval is unused.
+ ## Fields:
+ # interval: scanning interval, in seconds.
+ # minPool: Minimum number of messages to keep in pool.
+ # minSend: Minimum number of messages above minPool before we consider
+ # sending.
+ # sendRate: Largest fraction of the pool to send at a time.
def __init__(self, location, interval=600, minPool=6, minSend=1,
sendRate=.7):
"""Create a new queue that yields a batch of message every 'interval'
@@ -416,6 +423,7 @@
self.sendRate = sendRate
def _getBatchSize(self):
+ "Helper method: returns the number of messages to send."
pool = self.count()
if pool >= (self.minPool + self.minSend):
sendable = pool - self.minPool
@@ -424,6 +432,7 @@
return 0
def getBatch(self):
+ "Returns a list of handles for the next batch of messages to send."
n = self._getBatchSize()
if n:
return self.pickRandom(n)
@@ -442,18 +451,21 @@
if self.rng.getFloat() < msgProbability ])
def _secureDelete_bg(files, cleanFile):
+ """Helper method: delete files in another thread, removing 'cleanFile'
+ once we're done."""
+
pid = os.fork()
if pid != 0:
return pid
# Now we're in the child process.
try:
secureDelete(files, blocking=1)
- except OSError, _:
+ except OSError:
# This is sometimes thrown when shred finishes before waitpid.
pass
try:
os.unlink(cleanFile)
- except OSError, _:
+ except OSError:
pass
os._exit(0)
return None # Never reached.
Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- ServerInfo.py 7 Dec 2002 04:03:35 -0000 1.22
+++ ServerInfo.py 9 Dec 2002 04:47:40 -0000 1.23
@@ -15,7 +15,7 @@
import base64
import socket
-from mixminion.Common import createPrivateDir, getLog, MixError
+from mixminion.Common import createPrivateDir, LOG, MixError
from mixminion.Modules import SWAP_FWD_TYPE, FWD_TYPE
from mixminion.Packet import IPV4Info
import mixminion.Config
@@ -80,7 +80,7 @@
def __init__(self, fname=None, string=None, assumeValid=0):
mixminion.Config._ConfigFile.__init__(self, fname, string, assumeValid)
- getLog().trace("Reading server descriptor %s from %s",
+ LOG.trace("Reading server descriptor %s from %s",
self['Server']['Nickname'],
fname or "<string>")
@@ -120,18 +120,23 @@
#### XXXX001 CHECK OTHER SECTIONS
def getNickname(self):
+ """Returns this server's nickname"""
return self['Server']['Nickname']
def getAddr(self):
+ """Returns this server's IP address"""
return self['Server']['IP']
def getPort(self):
+ """Returns this server's IP port"""
return self['Incoming/MMTP']['Port']
def getPacketKey(self):
+ """Returns the RSA key this server uses to decrypt messages"""
return self['Server']['Packet-Key']
def getKeyID(self):
+ """Returns a hash of this server's MMTP key"""
return self['Incoming/MMTP']['Key-Digest']
def getRoutingInfo(self):
@@ -146,14 +151,24 @@
A server has one long-lived identity key, and two short-lived
temporary keys: one for subheader encryption and one for MMTP. The
subheader (or 'packet') key has an associated hashlog, and the
- MMTP key has an associated self-signed certificate.
+ MMTP key has an associated self-signed X509 certificate.
Whether we publish or not, we always generate a server descriptor
to store the keys' lifetimes.
When we create a new ServerKeyset object, the associated keys are not
read from disk unil the object's load method is called."""
+ ## Fields:
+ # hashlogFile: filename of this keyset's hashlog.
+ # packetKeyFile, mmtpKeyFile: filename of this keyset's short-term keys
+ # certFile: filename of this keyset's X509 certificate
+ # descFile: filename of this keyset's server descriptor.
+ #
+ # packetKey, mmtpKey: This server's actual short-term keys.
def __init__(self, keyroot, keyname, hashroot):
+ """Load a set of keys named "keyname" on a server where all keys
+ are stored under the directory "keyroot" and hashlogs are stored
+ under "hashroot". """
keydir = os.path.join(keyroot, "key_"+keyname)
self.hashlogFile = os.path.join(hashroot, "hash_"+keyname)
self.packetKeyFile = os.path.join(keydir, "mix.key")
@@ -164,13 +179,14 @@
createPrivateDir(keydir)
def load(self, password=None):
- "Read this set of keys from disk."
+ """Read the short-term keys from disk. Must be called before
+ getPacketKey or getMMTPKey."""
self.packetKey = mixminion.Crypto.pk_PEM_load(self.packetKeyFile,
password)
self.mmtpKey = mixminion.Crypto.pk_PEM_load(self.mmtpKeyFile,
password)
def save(self, password=None):
- "Save this set of keys to disk."
+ """Save this set of keys to disk."""
mixminion.Crypto.pk_PEM_save(self.packetKey, self.packetKeyFile,
password)
mixminion.Crypto.pk_PEM_save(self.mmtpKey, self.mmtpKeyFile,
@@ -189,6 +205,7 @@
return base64.encodestring(s).replace("\n", "")
def _time(t):
+ #XXXX001 move this to common.
"""Helper function: turns a time (in seconds) into the format used by
Server descriptors"""
gmt = time.gmtime(t)
@@ -196,12 +213,14 @@
gmt[0],gmt[1],gmt[2], gmt[3],gmt[4],gmt[5])
def _date(t):
+ #XXXX001 move this to common.
"""Helper function: turns a time (in seconds) into a date in the format
used by server descriptors"""
gmt = time.gmtime(t+1) # Add 1 to make sure we round down.
return "%04d/%02d/%02d" % (gmt[0],gmt[1],gmt[2])
def _rule(allow, (ip, mask, portmin, portmax)):
+ """Return an external represenntation of an IP allow/deny rule."""
if mask == '0.0.0.0':
ip="*"
mask=""
@@ -232,14 +251,19 @@
"""Generate and sign a new server descriptor, and generate all the keys to
go with it.
+ config -- Our ServerConfig object.
identityKey -- This server's private identity key
keydir -- The root directory for storing key sets.
keyname -- The name of this new key set within keydir
+ hashdir -- The root directory for storing hash logs.
validAt -- The starting time (in seconds) for this key's lifetime."""
+ # First, we generate both of our short-term keys...
packetKey = mixminion.Crypto.pk_generate(PACKET_KEY_BYTES*8)
mmtpKey = mixminion.Crypto.pk_generate(PACKET_KEY_BYTES*8)
+ # ...and save them to disk, setting up our directory structure while
+ # we're at it.
serverKeys = ServerKeyset(keydir, keyname, hashdir)
serverKeys.packetKey = packetKey
serverKeys.mmtpKey = mmtpKey
@@ -248,25 +272,27 @@
# FFFF unused
# allowIncoming = config['Incoming/MMTP'].get('Enabled', 0)
+ # Now, we pull all the information we need from our configuration.
nickname = config['Server']['Nickname']
if not nickname:
nickname = socket.gethostname()
if not nickname or nickname.lower().startswith("localhost"):
nickname = config['Incoming/MMTP'].get('IP', "<Unknown host>")
- getLog().warn("No nickname given: defaulting to %r", nickname)
+ LOG.warn("No nickname given: defaulting to %r", nickname)
contact = config['Server']['Contact-Email']
comments = config['Server']['Comments']
if not validAt:
validAt = time.time()
-
- # Round validAt to previous mignight.
+
+ # Calculate descriptor and X509 certificate lifetimes.
+ # (Round validAt to previous mignight.)
validAt = mixminion.Common.previousMidnight(validAt+30)
-
validUntil = validAt + config['Server']['PublicKeyLifetime'][2]
certStarts = validAt - CERTIFICATE_EXPIRY_SLOPPINESS
certEnds = validUntil + CERTIFICATE_EXPIRY_SLOPPINESS + \
config['Server']['PublicKeySloppiness'][2]
+ # Create the X509 certificate.
mixminion.Crypto.generate_cert(serverKeys.getCertFileName(),
mmtpKey,
"MMTP certificate for %s" %nickname,
@@ -287,14 +313,17 @@
_base64(serverKeys.getMMTPKeyID()),
}
+ # If we don't know our IP address, try to guess
if fields['IP'] == '0.0.0.0':
try:
fields['IP'] = _guessLocalIP()
- getLog().warn("No IP configured; guessing %s",fields['IP'])
+ LOG.warn("No IP configured; guessing %s",fields['IP'])
except IPGuessError, e:
- getLog().error("Can't guess IP: %s", str(e))
+ LOG.error("Can't guess IP: %s", str(e))
raise MixError("Can't guess IP: %s" % str(e))
+ # Fill in a stock server descriptor. Note the empty Digest: and
+ # Signature: lines.
info = """\
[Server]
Descriptor-Version: 0.1
@@ -313,6 +342,7 @@
if comments:
info += "Comments: %s\n"%comments
+ # Only advertise incoming MMTP if we support it.
if config["Incoming/MMTP"].get("Enabled", 0):
info += """\
[Incoming/MMTP]
@@ -326,6 +356,7 @@
continue
info += "%s: %s" % (k, _rule(k=='Allow',v))
+ # Only advertise outgoing MMTP if we support it.
if config["Outgoing/MMTP"].get("Enabled", 0):
info += """\
[Outgoing/MMTP]
@@ -337,9 +368,10 @@
continue
info += "%s: %s" % (k, _rule(k=='Allow',v))
+ # Ask our modules for their configuration information.
info += "".join(config.moduleManager.getServerInfoBlocks())
- # Remove extra (leading) whitespace.
+ # Remove extra (leading or trailing) whitespace from the lines.
lines = [ line.strip() for line in info.split("\n") ]
# Remove empty lines
lines = filter(None, lines)
@@ -348,6 +380,7 @@
info = "\n".join(lines)
info = signServerInfo(info, identityKey)
+ # Write the desciptor
f = open(serverKeys.getDescriptorFileName(), 'w')
try:
f.write(info)
@@ -412,10 +445,11 @@
return "\n".join(infoLines)
-
class IPGuessError(MixError):
+ """Exception: raised when we can't guess a single best IP."""
pass
+# Cached guessed IP address
_GUESSED_IP = None
def _guessLocalIP():
@@ -429,10 +463,10 @@
ip_set = {}
try:
ip_set[ socket.gethostbyname(socket.gethostname()) ] = 1
- except socket.error, _:
+ except socket.error:
try:
ip_set[ socket.gethostbyname(socket.getfqdn()) ] = 1
- except socket.error, _:
+ except socket.error:
pass
# And in case that doesn't work, let's see what other addresses we might
@@ -445,7 +479,7 @@
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((target_addr, 9)) #discard port
ip_set[ s.getsockname()[0] ] = 1
- except socket.error, _:
+ except socket.error:
pass
for ip in ip_set.keys():
Index: ServerMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerMain.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- ServerMain.py 7 Dec 2002 04:03:35 -0000 1.17
+++ ServerMain.py 9 Dec 2002 04:47:40 -0000 1.18
@@ -4,10 +4,10 @@
"""mixminion.ServerMain
The main loop and related functionality for a Mixminion server.
- See the "MixminionServer" class for more information about how it
- all works.
- BUG: No support for encrypting private keys."""
+ See the "MixminionServer" class for more information about how it
+ all works. """
+#FFFF We need support for encrypting private keys.
import os
import getopt
@@ -21,33 +21,18 @@
import mixminion.MMTPServer
from mixminion.ServerInfo import ServerKeyset, ServerInfo, _date, _time, \
generateServerDescriptorAndKeys
-from mixminion.Common import getLog, MixFatalError, MixError, secureDelete, \
+from mixminion.Common import LOG, MixFatalError, MixError, secureDelete, \
createPrivateDir, previousMidnight, ceilDiv
-# Directory layout:
-# MINION_HOME/work/queues/incoming/
-# mix/
-# outgoing/
-# deliver/mbox/
-# tls/dhparam
-# hashlogs/hash_1 ...
-# log
-# keys/identity.key
-# key_1/ServerDesc
-# mix.key
-# mmtp.key
-# mmtp.cert
-# key_2/...
-# conf/miniond.conf
-# ....
class ServerKeyring:
- """A ServerKeyRing remembers current and future keys, descriptors, and
+ """A ServerKeyring remembers current and future keys, descriptors, and
hash logs for a mixminion server.
FFFF We need a way to generate keys as needed, not just a month's
FFFF worth of keys up front.
"""
+ ## Fields:
# homeDir: server home directory
# keyDir: server key directory
# keySloppiness: fudge-factor: how forgiving are we about key liveness?
@@ -57,6 +42,27 @@
# keyRange: tuple of (firstKey, lastKey) to represent which key names
# have keys on disk.
+ ## Directory layout:
+ # MINION_HOME/work/queues/incoming/ [Queue of received,unprocessed pkts]
+ # mix/ [Mix pool]
+ # outgoing/ [Messages for mmtp delivery]
+ # deliver/mbox/ []
+ # tls/dhparam [Diffie-Hellman parameters]
+ # hashlogs/hash_1* [HashLogs of packet hashes
+ # hash_2* corresponding to key sets]
+ # ...
+ # log [Messages from the server]
+ # keys/identity.key [Long-lived identity PK]
+ # key_1/ServerDesc [Server descriptor]
+ # mix.key [packet key]
+ # mmtp.key [mmtp key]
+ # mmtp.cert [mmmtp key x509 cert]
+ # key_2/...
+ # conf/miniond.conf [configuration file]
+ # ....
+
+ # FFFF Support to put keys/queues in separate directories.
+
def __init__(self, config):
"Create a ServerKeyring from a config object"
self.configure(config)
@@ -77,41 +83,47 @@
firstKey = sys.maxint
lastKey = 0
- getLog().debug("Scanning server keystore at %s", self.keyDir)
+ LOG.debug("Scanning server keystore at %s", self.keyDir)
if not os.path.exists(self.keyDir):
- getLog().info("Creating server keystore at %s", self.keyDir)
+ LOG.info("Creating server keystore at %s", self.keyDir)
createPrivateDir(self.keyDir)
+ # Iterate over the entires in HOME/keys
for dirname in os.listdir(self.keyDir):
+ # Skip any that aren't directories named "key_INT"
if not os.path.isdir(os.path.join(self.keyDir,dirname)):
continue
if not dirname.startswith('key_'):
- getLog().warn("Unexpected directory %s under %s",
+ LOG.warn("Unexpected directory %s under %s",
dirname, self.keyDir)
continue
keysetname = dirname[4:]
try:
setNum = int(keysetname)
+ # keep trace of the first and last used key number
if setNum < firstKey: firstKey = setNum
if setNum > lastKey: lastKey = setNum
- except ValueError, _:
- getLog().warn("Unexpected directory %s under %s",
+ except ValueError:
+ LOG.warn("Unexpected directory %s under %s",
dirname, self.keyDir)
continue
+ # Find the server descriptor...
d = os.path.join(self.keyDir, dirname)
si = os.path.join(d, "ServerDesc")
if os.path.exists(si):
inf = ServerInfo(fname=si, assumeValid=1)
+ # And find out when it's valid.
t1 = inf['Server']['Valid-After']
t2 = inf['Server']['Valid-Until']
self.keyIntervals.append( (t1, t2, keysetname) )
- getLog().debug("Found key %s (valid from %s to %s)",
+ LOG.debug("Found key %s (valid from %s to %s)",
dirname, _date(t1), _date(t2))
else:
- getLog().warn("No server descriptor found for key %s"%dirname)
+ LOG.warn("No server descriptor found for key %s"%dirname)
+ # Now, sort the key intervals by starting time.
self.keyIntervals.sort()
self.keyRange = (firstKey, lastKey)
@@ -121,10 +133,10 @@
end = self.keyIntervals[idx][1]
start = self.keyIntervals[idx+1][0]
if start < end:
- getLog().warn("Multiple keys for %s. That's unsupported.",
+ LOG.warn("Multiple keys for %s. That's unsupported.",
_date(end))
elif start > end:
- getLog().warn("Gap in key schedule: no key from %s to %s",
+ LOG.warn("Gap in key schedule: no key from %s to %s",
_date(end), _date(start))
self.nextKeyRotation = 0 # Make sure that now > nextKeyRotation before
@@ -134,22 +146,21 @@
def getIdentityKey(self):
"""Return this server's identity key. Generate one if it doesn't
exist."""
- # FFFF Use this, somehow.
- password = None
+ password = None # FFFF Use this, somehow.
fn = os.path.join(self.keyDir, "identity.key")
bits = self.config['Server']['IdentityKeyBits']
if os.path.exists(fn):
key = mixminion.Crypto.pk_PEM_load(fn, password)
keylen = key.get_modulus_bytes()*8
if keylen != bits:
- getLog().warn(
+ LOG.warn(
"Stored identity key has %s bits, but you asked for %s.",
keylen, bits)
else:
- getLog().info("Generating identity key. (This may take a while.)")
+ LOG.info("Generating identity key. (This may take a while.)")
key = mixminion.Crypto.pk_generate(bits)
mixminion.Crypto.pk_PEM_save(key, fn, password)
- getLog().info("Generated %s-bit identity key.", bits)
+ LOG.info("Generated %s-bit identity key.", bits)
return key
@@ -157,16 +168,16 @@
"""Remove this server's identity key."""
fn = os.path.join(self.keyDir, "identity.key")
if not os.path.exists(fn):
- getLog().info("No identity key to remove.")
+ LOG.info("No identity key to remove.")
else:
- getLog().warn("Removing identity key in 10 seconds")
+ LOG.warn("Removing identity key in 10 seconds")
time.sleep(10)
- getLog().warn("Removing identity key")
+ LOG.warn("Removing identity key")
secureDelete([fn], blocking=1)
dhfile = os.path.join(self.homeDir, 'work', 'tls', 'dhparam')
if os.path.exists('dhfile'):
- getLog().info("Removing diffie-helman parameters file")
+ LOG.info("Removing diffie-helman parameters file")
secureDelete([dhfile], blocking=1)
def createKeys(self, num=1, startAt=None):
@@ -201,7 +212,7 @@
nextStart = startAt + self.config['Server']['PublicKeyLifetime'][2]
- getLog().info("Generating key %s to run from %s through %s (GMT)",
+ LOG.info("Generating key %s to run from %s through %s (GMT)",
keyname, _date(startAt), _date(nextStart-3600))
generateServerDescriptorAndKeys(config=self.config,
identityKey=self.getIdentityKey(),
@@ -228,7 +239,7 @@
for va, vu, name in self.keyIntervals if vu < cutoff ]
for dirname, (va, vu, name) in zip(dirs, self.keyIntervals):
- getLog().info("Removing%s key %s (valid from %s through %s)",
+ LOG.info("Removing%s key %s (valid from %s through %s)",
expiryStr, name, _date(va), _date(vu-3600))
files = [ os.path.join(dirname,f)
for f in os.listdir(dirname) ]
@@ -278,11 +289,11 @@
createPrivateDir(dhdir)
dhfile = os.path.join(dhdir, 'dhparam')
if not os.path.exists(dhfile):
- getLog().info("Generating Diffie-Helman parameters for TLS...")
+ LOG.info("Generating Diffie-Helman parameters for TLS...")
mixminion._minionlib.generate_dh_parameters(dhfile, verbose=0)
- getLog().info("...done")
+ LOG.info("...done")
else:
- getLog().debug("Using existing Diffie-Helman parameter from %s",
+ LOG.debug("Using existing Diffie-Helman parameter from %s",
dhfile)
return dhfile
@@ -320,7 +331,7 @@
def queueMessage(self, msg):
"""Add a message for delivery"""
- getLog().trace("Inserted message %r into incoming queue", msg[:8])
+ LOG.trace("Inserted message %r into incoming queue", msg[:8])
self.queueDeliveryMessage(None, msg)
def _deliverMessages(self, msgList):
@@ -331,22 +342,22 @@
res = ph.processMessage(message)
if res is None:
# Drop padding before it gets to the mix.
- getLog().debug("Padding message %r dropped",
+ LOG.debug("Padding message %r dropped",
message[:8])
else:
- getLog().debug("Processed message %r; inserting into pool",
+ LOG.debug("Processed message %r; inserting into pool",
message[:8])
self.mixPool.queueObject(res)
self.deliverySucceeded(handle)
except mixminion.Crypto.CryptoError, e:
- getLog().warn("Invalid PK or misencrypted packet header: %s",
+ LOG.warn("Invalid PK or misencrypted packet header: %s",
e)
self.deliveryFailed(handle)
except mixminion.Packet.ParseError, e:
- getLog().warn("Malformed message dropped: %s", e)
+ LOG.warn("Malformed message dropped: %s", e)
self.deliveryFailed(handle)
except mixminion.PacketHandler.ContentError, e:
- getLog().warn("Discarding bad packet: %s", e)
+ LOG.warn("Discarding bad packet: %s", e)
self.deliveryFailed(handle)
class MixPool:
@@ -376,19 +387,19 @@
"""Get a batch of messages, and queue them for delivery as
appropriate."""
handles = self.queue.getBatch()
- getLog().debug("Mixing %s messages out of %s",
+ LOG.debug("Mixing %s messages out of %s",
len(handles), self.queue.count())
for h in handles:
tp, info = self.queue.getObject(h)
if tp == 'EXIT':
rt, ri, app_key, tag, payload = info
- getLog().debug(" (sending message %r to exit modules)",
+ LOG.debug(" (sending message %r to exit modules)",
payload[:8])
self.moduleManager.queueMessage(payload, tag, rt, ri)
else:
assert tp == 'QUEUE'
ipv4, msg = info
- getLog().debug(" (sending message %r to MMTP server)",
+ LOG.debug(" (sending message %r to MMTP server)",
msg[:8])
self.outgoingQueue.queueDeliveryMessage(ipv4, msg)
self.queue.removeMessage(h)
@@ -417,11 +428,11 @@
self.server.sendMessages(addr.ip, addr.port, addr.keyinfo,
list(messages), list(handles))
-class _MMTPServer(mixminion.MMTPServer.MMTPServer):
+class _MMTPServer(mixminion.MMTPServer.MMTPAsyncServer):
"""Implementation of mixminion.MMTPServer that knows about
delivery queues."""
def __init__(self, config, tls):
- mixminion.MMTPServer.MMTPServer.__init__(self, config, tls)
+ mixminion.MMTPServer.MMTPAsyncServer.__init__(self, config, tls)
def connectQueues(self, incoming, outgoing):
self.incomingQueue = incoming
@@ -439,29 +450,48 @@
class MixminionServer:
"""Wraps and drives all the queues, and the async net server. Handles
all timed events."""
+ ## Fields:
+ # config: The ServerConfig object for this server
+ # keyring: The ServerKeyring
+ #
+ # mmtpServer: Instance of mixminion.ServerMain._MMTPServer. Receives
+ # and transmits packets from the network. Places the packets it
+ # receives in self.incomingQueue.
+ # incomingQueue: Instance of IncomingQueue. Holds received packets
+ # before they are decoded. Decodes packets with PacketHandler,
+ # and places them in mixPool.
+ # packetHandler: Instance of PacketHandler. Used by incomingQueue to
+ # decrypt, check, and re-pad received packets.
+ # mixPool: Instance of MixPool. Holds processed messages, and
+ # periodically decides which ones to deliver, according to some
+ # batching algorithm.
+ # moduleManager: Instance of ModuleManager. Map routing types to
+ # outging queues, and processes non-MMTP exit messages.
+ # outgoingQueue: Holds messages waiting to be send via MMTP.
+
def __init__(self, config):
"""Create a new server from a ServerConfig."""
- getLog().debug("Initializing server")
+ LOG.debug("Initializing server")
self.config = config
self.keyring = ServerKeyring(config)
if self.keyring._getLiveKey() is None:
- getLog().info("Generating a month's worth of keys.")
- getLog().info("(Don't count on this feature in future versions.)")
+ LOG.info("Generating a month's worth of keys.")
+ LOG.info("(Don't count on this feature in future versions.)")
# We might not be able to do this, if we password-encrypt keys
keylife = config['Server']['PublicKeyLifetime'][2]
nKeys = ceilDiv(30*24*60*60, keylife)
self.keyring.createKeys(nKeys)
- getLog().trace("Initializing packet handler")
+ LOG.trace("Initializing packet handler")
self.packetHandler = self.keyring.getPacketHandler()
- getLog().trace("Initializing TLS context")
+ LOG.trace("Initializing TLS context")
tlsContext = self.keyring.getTLSContext()
- getLog().trace("Initializing MMTP server")
+ LOG.trace("Initializing MMTP server")
self.mmtpServer = _MMTPServer(config, tlsContext)
# FFFF Modulemanager should know about async so it can patch in if it
# FFFF needs to.
- getLog().trace("Initializing delivery module")
+ LOG.trace("Initializing delivery module")
self.moduleManager = config.getModuleManager()
self.moduleManager.configure(config)
@@ -469,25 +499,25 @@
queueDir = os.path.join(homeDir, 'work', 'queues')
incomingDir = os.path.join(queueDir, "incoming")
- getLog().trace("Initializing incoming queue")
+ LOG.trace("Initializing incoming queue")
self.incomingQueue = IncomingQueue(incomingDir, self.packetHandler)
- getLog().trace("Found %d pending messages in incoming queue",
+ LOG.trace("Found %d pending messages in incoming queue",
self.incomingQueue.count())
mixDir = os.path.join(queueDir, "mix")
# FFFF The choice of mix algorithm should be configurable
- getLog().trace("Initializing Mix pool")
+ LOG.trace("Initializing Mix pool")
self.mixPool = MixPool(mixminion.Queue.TimedMixQueue(mixDir, 60))
- getLog().trace("Found %d pending messages in Mix pool",
+ LOG.trace("Found %d pending messages in Mix pool",
self.mixPool.count())
outgoingDir = os.path.join(queueDir, "outgoing")
- getLog().trace("Initializing outgoing queue")
+ LOG.trace("Initializing outgoing queue")
self.outgoingQueue = OutgoingQueue(outgoingDir)
- getLog().trace("Found %d pending messages in outgoing queue",
+ LOG.trace("Found %d pending messages in outgoing queue",
self.outgoingQueue.count())
- getLog().trace("Connecting queues")
+ LOG.trace("Connecting queues")
self.incomingQueue.connectQueues(mixPool=self.mixPool)
self.mixPool.connectQueues(outgoing=self.outgoingQueue,
manager=self.moduleManager)
@@ -506,7 +536,7 @@
#FFFF Unused
#nextRotate = self.keyring.getNextKeyRotation()
while 1:
- getLog().trace("Next mix at %s", _time(nextMix))
+ LOG.trace("Next mix at %s", _time(nextMix))
while time.time() < nextMix:
# Handle pending network events
self.mmtpServer.process(1)
@@ -518,7 +548,7 @@
# FFFF We need to recover on server failure.
self.packetHandler.syncLogs()
- getLog().trace("Mix interval elapsed")
+ LOG.trace("Mix interval elapsed")
# Choose a set of outgoing messages; put them in outgoingqueue and
# modulemanger
self.mixPool.mix()
@@ -533,7 +563,7 @@
if now > nextShred:
# FFFF Configurable shred interval
- getLog().trace("Expunging deleted messages from queues")
+ LOG.trace("Expunging deleted messages from queues")
self.incomingQueue.cleanQueue()
self.mixPool.queue.cleanQueue()
self.outgoingQueue.cleanQueue()
@@ -545,7 +575,6 @@
self.packetHandler.close()
#----------------------------------------------------------------------
-
def usageAndExit(cmd):
executable = sys.argv[0]
print >>sys.stderr, "Usage: %s %s [-h] [-f configfile]" % (executable, cmd)
@@ -581,29 +610,29 @@
def runServer(cmd, args):
config = configFromServerArgs(cmd, args)
try:
- mixminion.Common.getLog().configure(config)
- getLog().debug("Configuring server")
+ mixminion.Common.LOG.configure(config)
+ LOG.debug("Configuring server")
mixminion.Common.configureShredCommand(config)
mixminion.Crypto.init_crypto(config)
server = MixminionServer(config)
except:
- getLog().fatal_exc(sys.exc_info(),"Exception while configuring server")
+ LOG.fatal_exc(sys.exc_info(),"Exception while configuring server")
print >>sys.stderr, "Shutting down because of exception"
#XXXX print stack trace as well as logging?
sys.exit(1)
- getLog().info("Starting server")
+ LOG.info("Starting server")
try:
server.run()
except KeyboardInterrupt:
pass
except:
- getLog().fatal_exc(sys.exc_info(),"Exception while running server")
+ LOG.fatal_exc(sys.exc_info(),"Exception while running server")
#XXXX print stack trace as well as logging?
- getLog().info("Server shutting down")
+ LOG.info("Server shutting down")
server.close()
- getLog().info("Server is shut down")
+ LOG.info("Server is shut down")
sys.exit(0)
@@ -625,7 +654,7 @@
elif opt in ('-n', '--keys'):
try:
keys = int(val)
- except ValueError, _:
+ except ValueError:
print >>sys.stderr,("%s requires an integer" %opt)
usage = 1
if usage:
@@ -634,7 +663,7 @@
config = readConfigFile(configFile)
- getLog().setMinSeverity("INFO")
+ LOG.setMinSeverity("INFO")
mixminion.Crypto.init_crypto(config)
keyring = ServerKeyring(config)
print >>sys.stderr, "Creating %s keys..." % keys
@@ -669,11 +698,11 @@
config = readConfigFile(configFile)
mixminion.Common.configureShredCommand(config)
- getLog().setMinSeverity("INFO")
+ LOG.setMinSeverity("INFO")
keyring = ServerKeyring(config)
keyring.checkKeys()
# This is impossibly far in the future.
keyring.removeDeadKeys(now=(1L << 36))
if removeIdentity:
keyring.removeIdentityKey()
- getLog().info("Done removing keys")
+ LOG.info("Done removing keys")
Index: __init__.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/__init__.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- __init__.py 2 Dec 2002 03:30:07 -0000 1.7
+++ __init__.py 9 Dec 2002 04:47:40 -0000 1.8
@@ -4,8 +4,7 @@
"""mixminion
Client and server code for type III anonymous remailers.
-
- DOCDOC write more on principal interfaces"""
+ """
__version__ = "0.0.1a0"
__all__ = [ ]
Index: benchmark.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/benchmark.py,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- benchmark.py 2 Dec 2002 20:18:44 -0000 1.14
+++ benchmark.py 9 Dec 2002 04:47:40 -0000 1.15
@@ -105,7 +105,7 @@
shakey = "8charstr"*2
print "Keyed SHA1 for lioness (28K, unoptimized)", timeit(
- (lambda shakey=shakey: _ml.sha1("".join([shakey,s28K,shakey]))), 1000)
+ (lambda shakey=shakey: _ml.sha1("".join((shakey,s28K,shakey)))), 1000)
print "TRNG (20 byte)", timeit((lambda: trng(20)), 100)
print "TRNG (128 byte)", timeit((lambda: trng(128)), 100)
@@ -406,7 +406,12 @@
#----------------------------------------------------------------------
def timeEfficiency():
print "#================= ACTUAL v. IDEAL ====================="
-
+ # Here we compare the time spent in an operation with the time we think
+ # is required for its underlying operations, in order to try to measure
+ # its efficiency. If function X is pretty efficient, there's not much
+ # reason to try to optimise its implementation; instead, we need to attack
+ # the functions it uses.
+
##### LIONESS
shakey = "z"*20
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.42
retrieving revision 1.43
diff -u -d -r1.42 -r1.43
--- test.py 7 Dec 2002 04:03:35 -0000 1.42
+++ test.py 9 Dec 2002 04:47:40 -0000 1.43
@@ -24,25 +24,32 @@
import cPickle
import cStringIO
+# Not every post-2.0 version of Python has a working 'unittest' module, so
+# we include a copy with mixminion, as 'mixminion._unittest'.
+try:
+ import unittest
+except ImportError:
+ import mixminion._unittest as unittest
+
import mixminion.testSupport
[...1158 lines suppressed...]
-
suite.addTest(tc(MiscTests))
suite.addTest(tc(MinionlibCryptoTests))
suite.addTest(tc(CryptoTests))
@@ -3553,12 +3754,12 @@
init_crypto()
# Suppress 'files-can't-be-securely-deleted' message while testing
- getLog().setMinSeverity("FATAL")
+ LOG.setMinSeverity("FATAL")
mixminion.Common.secureDelete([],1)
# Disable TRACE and DEBUG log messages, unless somebody overrides from
# the environment.
- getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
- #getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "TRACE"))
+ LOG.setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
+ #LOG.setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "TRACE"))
unittest.TextTestRunner(verbosity=1).run(testSuite())
Index: testSupport.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/testSupport.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- testSupport.py 2 Dec 2002 03:25:46 -0000 1.8
+++ testSupport.py 9 Dec 2002 04:47:40 -0000 1.9
@@ -3,7 +3,7 @@
"""mixminion.testSupport
- Shared support code for unit tests, benchmark tests, and integration tests
+ Shared support code for unit tests, benchmark tests, and integration tests.
"""
import os
@@ -12,13 +12,17 @@
import base64
import cStringIO
+import mixminion.Crypto
import mixminion.Common
-from mixminion.Common import waitForChildren, createPrivateDir, getLog
+from mixminion.Common import waitForChildren, createPrivateDir, LOG
from mixminion.Config import _parseBoolean, ConfigError
from mixminion.Modules import DeliveryModule, ImmediateDeliveryQueue, \
SimpleModuleDeliveryQueue, DELIVER_OK, DELIVER_FAIL_RETRY, \
DELIVER_FAIL_NORETRY, _escapeMessageForEmail
+#----------------------------------------------------------------------
+# DirectoryStoreModule
+
class DirectoryStoreModule(DeliveryModule):
"""Delivery module for testing: puts messages in files in a given
directory. Can be configured to use a delivery queue or not.
@@ -30,6 +34,11 @@
Otherwise, creates a file in the specified directory, containing
the routing info, a newline, and the message contents.
"""
+ ## Fields:
+ # loc -- The directory to store files in. All filenames are numbers;
+ # we always put new messages in the smallest number greater than
+ # all existing numbers.
+ # next -- the number of the next file.
def getConfigSyntax(self):
return { 'Testing/DirectoryDump':
{ 'Location' : ('REQUIRE', None, None),
@@ -78,11 +87,12 @@
elif exitInfo == 'FAIL!':
return DELIVER_FAIL_NORETRY
- getLog().debug("Delivering test message")
+ LOG.debug("Delivering test message")
m = _escapeMessageForEmail(message, tag)
if m is None:
- # Ordinarily, we'd drop these.
+ # Ordinarily, we'd drop corrupt messages, but this module is
+ # meant for debugging.
m = """\
==========CORRUPT OR UNDECODABLE MESSAGE
Decoding handle: %s%s==========MESSAGE ENDS""" % (
@@ -96,12 +106,19 @@
return DELIVER_OK
#----------------------------------------------------------------------
+# mix_mktemp: A secure, paranoid mktemp replacement. (May be overkill
+# for testing, but better safe than sorry.)
-# Test for acceptable permissions and uid on directory?
+# Constant flag: are we paranoid about permissions and uid on our tmpdir?
_MM_TESTING_TEMPDIR_PARANOIA = 1
-# Holds
+# Name of our temporary directory: all temporary files go under this
+# directory. If None, it hasn't been created yet. If it exists,
+# it must be owned by us, mode 700, and have no parents that an adversary
+# (other than root) could write to.
_MM_TESTING_TEMPDIR = None
+# How many temporary files have we created so far?
_MM_TESTING_TEMPDIR_COUNTER = 0
+# Do we nuke the contents of _MM_TESTING_TEMPDIR on exit?
_MM_TESTING_TEMPDIR_REMOVE_ON_EXIT = 1
def mix_mktemp(extra=""):
'''mktemp wrapper. puts all files under a securely mktemped
@@ -109,8 +126,12 @@
global _MM_TESTING_TEMPDIR
global _MM_TESTING_TEMPDIR_COUNTER
if _MM_TESTING_TEMPDIR is None:
+ # We haven't configured our temporary directory yet.
import tempfile
paranoia = _MM_TESTING_TEMPDIR_PARANOIA
+
+ # If tempfile.mkdtemp exists, use it. This avoids warnings, and
+ # is harder for people to exploit.
if hasattr(tempfile, 'mkdtemp'):
try:
temp = tempfile.mkdtemp()
@@ -118,6 +139,8 @@
print "mkdtemp failure: %s" % e
sys.exit(1)
else:
+ # Otherwise, pick a dirname, make sure it doesn't exist, and try to
+ # create it.
temp = tempfile.mktemp()
if paranoia and os.path.exists(temp):
print "I think somebody's trying to exploit mktemp."
@@ -128,18 +151,24 @@
print "Something's up with mktemp: %s" % e
sys.exit(1)
+ # The directory must exist....
if not os.path.exists(temp):
print "Couldn't create temp dir %r" %temp
sys.exit(1)
st = os.stat(temp)
if paranoia:
+ # And be writeable only by us...
if st[stat.ST_MODE] & 077:
print "Couldn't make temp dir %r with secure permissions" %temp
sys.exit(1)
+ # And be owned by us...
if st[stat.ST_UID] != os.getuid():
print "The wrong user owns temp dir %r"%temp
sys.exit(1)
parent = temp
+ # And if, and all of its parents, must not be group-writeable
+ # unless their sticky bit is set, and must not be owned by
+ # anybody except us and root.
while 1:
p = os.path.split(parent)[0]
if parent == p:
@@ -159,7 +188,9 @@
if _MM_TESTING_TEMPDIR_REMOVE_ON_EXIT:
import atexit
atexit.register(deltree, temp)
-
+
+ # So now we have a temporary directory; return the name of a new
+ # file there.
_MM_TESTING_TEMPDIR_COUNTER += 1
return os.path.join(_MM_TESTING_TEMPDIR,
"tmp%05d%s" % (_MM_TESTING_TEMPDIR_COUNTER,extra))
@@ -167,7 +198,7 @@
_WAIT_FOR_KIDS = 1
def deltree(*dirs):
"""Delete each one of a list of directories, along with all of its
- contents"""
+ contents."""
global _WAIT_FOR_KIDS
if _WAIT_FOR_KIDS:
print "Waiting for shred processes to finish."
@@ -186,10 +217,11 @@
os.unlink(d)
#----------------------------------------------------------------------
+# suspendLog
def suspendLog(severity=None):
- """Temporarily suppress logging output."""
- log = getLog()
+ """Temporarily suppress logging output"""
+ log = LOG
if hasattr(log, '_storedHandlers'):
resumeLog()
buf = cStringIO.StringIO()
@@ -205,7 +237,7 @@
def resumeLog():
"""Resume logging output. Return all new log messages since the last
suspend."""
- log = getLog()
+ log = LOG
if not hasattr(log, '_storedHandlers'):
return None
buf = log._testBuf
@@ -217,12 +249,14 @@
return buf.getvalue()
#----------------------------------------------------------------------
+# Facilities to temporarily replace attributes and functions for testing
-# list of obj, attr, oldval.
+# List of object, attribute, old-value for all replaced attributes.
_REPLACED_OBJECT_STACK = []
def replaceAttribute(object, attribute, value):
- "DOCDOC"
+ """Temporarily replace <object.attribute> with value. When
+ undoReplacedAttributes() is called, the old value is restored."""
if hasattr(object, attribute):
tup = (object, attribute, getattr(object, attribute))
else:
@@ -230,9 +264,13 @@
_REPLACED_OBJECT_STACK.append(tup)
setattr(object, attribute, value)
+# List of (fnname, args, kwargs) for all the replaced functions that
+# have been called.
_CALL_LOG = []
class _ReplacementFunc:
+ """Helper object: callable stub that logs its invocations to _CALL_LOG
+ and delegates to an internal function."""
def __init__(self, name, fn=None):
self.name = name
self.fn = fn
@@ -244,16 +282,24 @@
return None
def replaceFunction(object, attribute, fn=None):
+ """Temporarily replace the function or method <object.attribute>.
+ If <fn> is provided, replace it with fn; otherwise, the new
+ function will just return None. All invocations of the new
+ function will logged, and retrievable by getReplacedFunctionCallLog()"""
replaceAttribute(object, attribute, _ReplacementFunc(attribute, fn))
def getReplacedFunctionCallLog():
+ """Return a list of (functionname, args, kwargs)"""
return _CALL_LOG
def clearReplacedFunctionCallLog():
+ """Clear all entries from the replaced function call log"""
del _CALL_LOG[:]
def undoReplacedAttributes():
- "DOCDOC"
+ """Undo all replaceAttribute and replaceFunction calls in effect."""
+ # Remember to traverse _REPLACED_OBJECT_STACK in reverse, so that
+ # "replace(a,b,c1); replace(a,b,c2)" is properly undone.
r = _REPLACED_OBJECT_STACK[:]
r.reverse()
del _REPLACED_OBJECT_STACK[:]
@@ -264,3 +310,82 @@
else:
o,a,v = item
setattr(o,a,v)
+
+#----------------------------------------------------------------------
+# Long keypairs: stored here to avoid regenerating them every time we need
+# to run tests. (We can't use 1024-bit keys, since they're not long enough
+# to use as identity keys.)
+
+TEST_KEYS_2048 = [
+"""\
+MIIEowIBAAKCAQEA0aBBHqAyfoAweyq5NGozHezVut12lGHeKrfmnax9AVPMfueqskqcKsjMe3Rz
+NhDukD3ebYKPLKMnVDM+noVyHSawnzIc+1+wq1LFP5TJiPkPdodKq/SNlz363kkluLwhoWdn/16k
+jlprnvdDk6ZxuXXTsAGtg235pEtFs4BLOLOxikW2pdt2Tir71p9SY0zGdM8m5UWZw4z3KqYFfPLI
+oBsN+3hpcsjjO4BpkzpP3zVxy8VN2+hCxjbfow2sO6faD2u6r8BXPB7WlAbmwD8ZoX6f8Fbay02a
+jG0mxglE9f0YQr66DONEQPoxQt8C1gn3KAIQ2Hdw1cxpQf3lkceBywIDAQABAoIBAETRUm+Gce07
+ki7tIK4Ha06YsLXO/J3L306w3uHGfadQ5mKHFW/AtLILB65D1YrbViY+WWYkJXKnAUNQK2+JKaRO
+Tk+E+STBDlPAMYclBmCUOzJTSf1XpKARNemBpAOYp4XAV9DrNiSRpKEkVagETXNwLhWrB1aNZRY9
+q9048fjj1NoXsvLVY6HTaViHn8RCxuoSHT/1LXjStvR9tsLHk6llCtzcRO1fqBH7gRog8hhL1g5U
+rfUJnXNSC3C2P9bQty0XACq0ma98AwGfozrK3Ca40GtlqYbsNsbKHgEgSVe124XDeVweK8b56J/O
+EUsWF5hwdZnBTfmJP8IWmiXS16ECgYEA8YxFt0GrqlstLXEytApkkTZkGDf3D1Trzys2V2+uUExt
+YcoFrZxIGLk8+7BPHGixJjLBvMqMLNaBMMXH/9HfSyHN3QHXWukPqNhmwmnHiT00i0QsNsdwsGJE
+xXH0HsxgZCKDkLbYkzmzetfXPoaP43Q5feVSzhmBrZ3epwlTJDECgYEA3isKtLiISyGuao4bMT/s
+3sQcgqcLArpNiPUo5ESp5qbXflAiH2wTC1ZNh7wUtn0Am8TdG1JnKFUdwHELpiRP9yCQj2bFS/85
+jk6RCEmXdAGpYzB6lrqtYhFNe5LzphLGtALsuVOq6I7LQbUXY3019fkawfiFvnYZVovC3DKCsrsC
+gYBSg8y9EZ4HECaaw3TCtFoukRoYe+XWQvhbSTPDIs+1dqZXJaBS8nRenckLYetklQ8PMX+lcrv4
+BT8U3ju4VIWnMOEWgq6Cy+MhlutjtqcHZvUwLhW8kN0aJDfCC2+Npdu32WKAaTYK9Ucuy9Un8ufs
+l6OcMl7bMTNvj+KjxTe1wQKBgB1cSNTrUi/Dqr4wO429qfsipbXqh3z7zAVeiOHp5R4zTGVIB8pp
+SPcFl8dpZr9bM7piQOo8cJ+W6BCnn+d8Awlgx1n8NfS+LQgOgAI9X4OYOJ+AJ6NF1mYQbVH4cLSw
+5Iujm08+rGaBgIEVgprGUFxKaGvcASjTiLO0UrMxBa7DAoGBALIwOkPLvZNkyVclSIdgrcWURlyC
+oAK9MRgJPxS7s6KoJ3VXVKtIG3HCUXZXnmkPXWJshDBHmwsv8Zx50f+pqf7MD5fi3L1+rLjN/Rp/
+3lGmzcVrG4LO4FEgs22LXKYfpvYRvcsXzbwHX33LnyLeXKrKYQ82tdxKOrh9wnEDqDmh""",
+"""\
+MIIEpQIBAAKCAQEAv/fvw/2HK48bwjgR2nUQ1qea9eIsYv4m98+DQoqPO7Zlr+Qs6/uiiOKtH0/b
+3/B9As261HKkI4VDG0L523rB1QAfeENKdLczj8DoQPjHMsNDDepbBYmYH91vmig47fbLmbDnUiSD
++CFtM+/wUG4holomQBdPfUhoc44Fcw3cyvskkJr5aN9rqBRGuwuR81RaXt5lKtiwv9JUYqEBb2/f
+sSDEWWHSf9HemzR25M/T+A51yQwKyFXC4RQzCu2jX7sZ53c6KRCniLPq9wUwtTrToul34Sssnw8h
+PiV0Fwrk12uJdqqLDbltUlp6SEx8vBjSZC6JnVsunYmw88sIYGsrbQIDAQABAoIBAQCpnDaLxAUZ
+x2ePQlsD2Ur3XT7c4Oi2zjc/3Gjs8d97srxFnCTUm5APwbeUYsqyIZlSUNMxwdikSanw/EwmT1/T
+AjjL2Sh/1x4HdTm/rg7SGxOzx8yEJ/3wqYVhfwhNuDBLqrG3Mewn3+DMcsKxTZ0KBPymw/HHj6I5
+9tF5xlW+QH7udAPxAX3qZC/VveqlomGTu4rBBtGt1mIIt+iP4kjlOjIutb6EK3fXZ8r9VZllNJ3D
+/xZVx7Jt40hcV6CEuWOg1lwXQNmgl8+bSUvTaCpiVQ4ackeosWhTWxtKndw4UXSzXZAbjHAmAwMY
+bHwxN4AqZZfbb2EI1WzOBjeZje1BAoGBAOiQZgngJr++hqn0gJOUImv+OWpFMfffzhWyMM8yoPXK
+tIKaFTEuHAkCVre6lA1g34cFeYDcK9BC4oyQbdO4nxTZeTnrU2JQK2t4+N7WBU9W5/wOlxEdYzE0
+2rNrDxBtOtCQnOI1h9Mrc87+xzPP55OloKbRMW1JzeAxWdg1LJrvAoGBANNQRNdRzgoDAm0N7WNe
+pGx51v+UuGUHvE4dMGKWdK8njwbsv6l7HlTplGGOZUThZWM7Ihc8LU6NZ2IlwWYgzivYL/SUejUD
+9/rYaWEYWPdXQW2/ekdi3FFZtKcuUB5zLy3gqtLSjM1a8zhbxdkYq4tqa+v9JwMTr/oyVf//XM9j
+AoGAEjftpmxm3LKCPiInSGhcYfVibg7JoU9pB44UAMdIkLi2d1y2uEmSbKpAPNhi7MFgAWXOZOfa
+jtAOi1BtKh7WZ325322t9I+vNxYc+OfvNo3qUnaaIv8YXCx1zYRfg7vq1ZfekmH7J/HJere+xzJM
+Q+a/tRHCO3uCo0N6dFOGEQUCgYEAsQhJdD6zqA2XZbfKTnrGs55rsdltli6h4qtvktjLzsYMfFex
+xpI/+hFqX0TFsKxInZa329Ftf6bVmxNYcHBBadgHbRdLPskhYsUVm+Oi/Szbws8s6Ut4mqrVv038
+j1Yei4fydQcyMQTmSSwRl+ykIvu4iI+gtGI1Bx5OkFbm8VMCgYEAlEvig/fGBA/MgE6DUf6MXbFn
+92JW25az5REkpZtEXz3B6yhyt/S5D1Da6xvfqvNijyqZpUqtp7lPSOlqFRJ3NihNc8lRqyFMPiBn
+41QQWPZyFa1rTwJxijyG9PkI0sl1/WQK5QrTjGZGjX7r4Fjzr6EYM8gH3RA3WAPzJylTOdo=""",
+"""\
+MIIEpQIBAAKCAQEA68uqw2Ao12QPktY9pf9VSHMfJ8jKBGG4eG+HPmaBifc6+kAZWA7jeOwMTnbS
++KZ2nMFXKthp6zJiDzQqgKlQ7eA0zzBPtAboy4YhPRwrrQr/o1oPrppS2eEwvCGewySAZsIUwX4d
+0P68lpLbA9h1vuV3t19M2WNifsYYcTUGPGdbpZHgBDQdmQeUBkXtCTANPxOYsrLwEhaCBrK4BLkW
+sRNi0dRnFRdJ18rAYCiDAKq168IyP4TCUKKGWHbquv5rrNdg/RoUiCyPTgDodLaXTOLrRPuCOl5p
+dwhNSwJyzEpeqy/x4YnNRbGNv7M1sNhnrarbUduZqOz9RpTQ0niKFQIDAQABAoIBAQC2h1aNH2b+
+NWsI0+etFFbEWrmHZptbgPn34P3khB1K26NADVaRIBVeifuM0dbGvLWc6t27QQPdGYdnFY7BQlBv
+k9vNdyx7w815nz8juybkMVtq7FCvbK8uEnBTcgMgNKVg5mSC1Enoewkp1kzMUUf0mlVuEcu/jHu2
+f0p0eAN3xV5f4up+soujOrWuradmZ3uirYXzYrApagUHMqtjr+AhXJx7MuQCv9UPRU7ouidV/q36
+Q/C4OpRqizjiKzulLhUoHmAUGMEQOd+ICoy71HOiK4MqnCmt2vI34cV9Cd5A8Hlfm6/COseor0Sq
+26t4f8M8un7efc/RsF1xULiz/RoRAoGBAPvyQRyts6xpvDnanBLQa7b1Qf8oatYIcCcC7JlU+DZX
+wD5qroyE5O7xStnSjqX5D6Lc7RbINkAuNGCofJzzynl5tP9j0WREueT1nq/YUW7Xn+Pd0fD6Fgb4
+Js2vdRybH+vG4mv4gMxnS/gY+9jR7HL3GJRRQMMM5zWKY4LvrVADAoGBAO+W46I0/X5WCWellMod
+Pa0M9OY3a8pJyP//JzblYykgw5nWWPHZEEOxV4VGFP0Pz4i6kpq/psWbCNLsh9k7EsqWLpeE7wsW
+uXQj5LruIupL9/notboifL4zIOQcvHNs25iya+yURISYcVhmlqHHofX7ePfQR5sg1e1ZvethyR4H
+AoGBAOH1ZhIrc14pQmf8uUdiZ4iiM/t8qzykOrmyNLJb83UBhGg2U6+xLIVkIMZ0wfz2/+AIFhb9
+nzI2fkFGOuSk/S2vSvZV9qDfxn0jEJwS/Q3VExBRjA18ra64dky4lOb/9UQHjmBZcmJgLlEnTxAp
+Tc/Z7tBugw+sDd0F7bOr85szAoGAOOBzLaCyxPkbxnUye0Cx0ZEP2k8x0ZXul4c1Af02qx7SEIUo
+HFHRYKCLDGJ0vRaxx92yy/XPW33QfHIWVeWGMn2wldvC+7jrUbzroczCkShzt+ocqhFh160/k6eW
+vTgMcZV5tXIFSgz+a2P/Qmyn8ENAlmPle9gxsOTrByPxoKUCgYEA1raYnqI9nKWkZYMrEOHx7Sy3
+xCaKFSoc4nBxjJvZsSJ2aH6fJfMksPTisbYdSaXkGrb1fN2E7HxM1LsnbCyvXZsbMUV0zkk0Tzum
+qDVW03gO4AvOD9Ix5gdebdq8le0xfMUzDvAIG1ypM+oMdZ122bI/rsOpLkZ4EtmixFxJbpk="""
+]
+
+TEST_KEYS_2048 = [
+ mixminion.Crypto.pk_decode_private_key(base64.decodestring(s))
+ for s in TEST_KEYS_2048 ]
+del s