[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Made TODO reflect open spec issues; updates to reflect ...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv12628/lib/mixminion
Modified Files:
Common.py HashLog.py MMTPClient.py MMTPServer.py Queue.py
test.py
Added Files:
README
Log Message:
Made TODO reflect open spec issues; updates to reflect spec; more testing.
TODO file: remove completed items; add a list of what's still
blocking; push advanced SSL stuff off till release "0.2".
src/README, lib/mixminion/README: describe the purpose of each file.
Common.py, HashLog.py, Queue.py: refactor logging a bit
MMTPClient.py: Check hash of server's key. Change from LF to CRLF.
MMTPServer.py:
- Remember when each connection last saw action.
- Check hash of server's key.
- Change from LF to CRLF.
test.py:
- Add tests for PEM-encoded RSA keys.
- Made all the tests absolutely clean up /tmp after themselves.
- Added support for automatic generation of private keys and DH
parameters for server testing. The tests for MMTP can now run
on a machine other than my own.
- Test keyid checking
- Suppress log messages during testing.
--- NEW FILE: README ---
DIRECTORY: lib/mixminion/
Python implementation of the Mixminion anonymizing system.
__init__.py
Required by Python to make this group of files a package.
Crypto.py
Wrappers for low-level cryptographic functionality
Common.py
Helper routines for OS functionality, base exceptions, etc.
HashLog.py
Class to keep a record of packets that we've processed.
Modules.py
Registry of delievery modules, and implementations of the most
simple ones
Queue.py
A directory-based unordered message queue.
Packet.py
Data structures from mixminion spec, and routines to serialize
and unserialize them.
ServerInfo.py
Representation for a server's descriptor block
Config.py
Configuration of the local server.
BuildMessage.py
Functions to construct packets and reply blocks
PacketHandler.py
Class to unwrap and process packets.
MMTPClient.py
Blocking client-side implementation of the Mixminion Transfer protocol
MMTPServer.py
Nonblocking implementation of MMTP
benchmark.py
test.py
Unit/performance tests
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- Common.py 5 Jul 2002 19:50:27 -0000 1.7
+++ Common.py 9 Jul 2002 04:07:14 -0000 1.8
@@ -7,8 +7,7 @@
__all__ = [ 'MixError', 'MixFatalError', 'onReset', 'onTerminate',
'installSignalHandlers', 'secureDelete', 'secureRename',
- 'ceilDiv', 'floorDiv', 'debug', 'warn', 'info', 'error',
- 'fatal' ]
+ 'ceilDiv', 'floorDiv', 'getLog' ]
import os
import signal
@@ -129,14 +128,29 @@
def write(self, severity, message):
print >> self.file, "%s [%s] %s" % (_logtime(), severity, message)
+
+_SEVERITIES = { 'TRACE' : -2,
+ 'DEBUG' : -1,
+ 'INFO' : 0,
+ 'WARN' : 1,
+ 'ERROR': 2,
+ 'FATAL' : 3 }
+
class Log:
def __init__(self, minSeverity):
self.handlers = []
+ self.setMinSeverity(minSeverity)
+ onReset(self.reset)
+ onTerminate(self.close)
+
+ def setMinSeverity(self, minSeverity):
+ self.severity = _SEVERITIES.get(minSeverity, 1)
def addHandler(self, handler):
self.handlers.append(handler)
def reset(self):
+ # FFFF reload configuration information here?
for h in self.handlers:
h.reset()
@@ -145,12 +159,15 @@
h.close()
def log(self, severity, message, *args):
- # Check that severity is okay.
+ if _SEVERITIES.get(severity, 100) < self.severity:
+ return
+ m = message % args
for h in self.handlers:
- h.write(severity, message % args)
+ h.write(severity, m)
+ def trace(self, message, *args):
+ self.log("TRACE", message, *args)
def debug(self, message, *args):
- # Need a means to filter messages
self.log("DEBUG", message, *args)
def info(self, message, *args):
self.log("INFO", message, *args)
@@ -161,16 +178,19 @@
def fatal(self, message, *args):
self.log("FATAL", message, *args)
-_theLog = Log('DEBUG')
-_theLog.addHandler(ConsoleLogTarget(sys.stderr))
-# XXXX Configure the log for real
+_theLog = None
+
+def getLog():
+ """Return the MixMinion log object."""
+ global _theLog
+ if _theLog is None:
+ # XXXX Configure the log for real
+ _theLog = Log('DEBUG')
+ _theLog.addHandler(ConsoleLogTarget(sys.stderr))
+
+ return _theLog
+
-log = _theLog.log
-debug = _theLog.debug
-info = _theLog.info
-warn = _theLog.warn
-error = _theLog.error
-fatal = _theLog.fatal
#----------------------------------------------------------------------
# Signal handling
@@ -191,7 +211,6 @@
"""Given a 0-argument function fn, cause fn to be invoked when
this process next receives a SIGTERM."""
terminateHooks.append(fn)
-
def waitForChildren():
"""Wait until all subprocesses have finished. Useful for testing."""
Index: HashLog.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/HashLog.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- HashLog.py 5 Jul 2002 19:50:27 -0000 1.8
+++ HashLog.py 9 Jul 2002 04:07:14 -0000 1.9
@@ -6,7 +6,7 @@
Persistant memory for the hashed secrets we've seen."""
import anydbm, dumbdbm
-from mixminion.Common import MixFatalError, warn
+from mixminion.Common import MixFatalError, getLog
__all__ = [ 'HashLog' ]
@@ -39,7 +39,7 @@
'keyid'."""
self.log = anydbm.open(filename, 'c')
if isinstance(self.log, dumbdbm._Database):
- warn("Warning: logging packet digests to a flat file.")
+ getLog().warn("Warning: logging packet digests to a flat file.")
try:
if self.log["KEYID"] != keyid:
raise MixFatalError("Log KEYID does not match current KEYID")
Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- MMTPClient.py 1 Jul 2002 18:03:05 -0000 1.3
+++ MMTPClient.py 9 Jul 2002 04:07:14 -0000 1.4
@@ -44,25 +44,28 @@
#XXXX session resumption
self.tls.connect()
peer_pk = self.tls.get_peer_cert_pk()
- # XXXX Check the key ... how exactly is this to be hashed?
+ keyID = sha1(self.tls.get_peer_cert_pk().encode_key(public=1))
+ if self.targetKeyID is not None and (keyID != self.targetKeyID):
+ raise MixProtocolError("Bad Key ID: Expected %r but got %r" % (
+ self.targetKeyID, keyID))
####
# Protocol negotiation
# For now, we only support 1.0
- self.tls.write("PROTOCOL 1.0\n")
- inp = self.tls.read(len("PROTOCOL 1.0\n"))
- if inp != "PROTOCOL 1.0\n":
+ self.tls.write("PROTOCOL 1.0\r\n")
+ inp = self.tls.read(len("PROTOCOL 1.0\r\n"))
+ if inp != "PROTOCOL 1.0\r\n":
raise MixProtocolError("Protocol negotiation failed")
def sendPacket(self, packet):
"""Send a single packet to a server."""
assert len(packet) == 1<<15
- self.tls.write("SEND\n")
+ self.tls.write("SEND\r\n")
self.tls.write(packet)
self.tls.write(sha1(packet+"SEND"))
- inp = self.tls.read(len("RECEIVED\n")+20)
- if inp != "RECEIVED\n"+sha1(packet+"RECEIVED"):
+ inp = self.tls.read(len("RECEIVED\r\n")+20)
+ if inp != "RECEIVED\r\n"+sha1(packet+"RECEIVED"):
raise MixProtocolError("Bad ACK received")
def shutdown(self):
Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- MMTPServer.py 5 Jul 2002 19:50:27 -0000 1.5
+++ MMTPServer.py 9 Jul 2002 04:07:14 -0000 1.6
@@ -16,31 +16,33 @@
# NOTE FOR THE CURIOUS: The 'asyncore' module in the standard library
# is another general select/poll wrapper... so why are we using our
-# own? Basically, because asyncore has IMO a couple of misdesigns,
-# the largest of which is that it has the 'server loop' periodically
-# query the connections for their status, whereas we have the
-# connections inform the server of their status whenever they
-# change. This latter approach turns out to be far easier to use
-# with TLS.
+# own? Basically, because asyncore has IMO a couple of mismatches
+# with our design, the largest of which is that it has the 'server
+# loop' periodically query the connections for their status,
+# whereas we have the connections inform the server of their status
+# whenever they change. This latter approach turns out to be far
+# easier to use with TLS.
import errno
import socket
import select
import re
-import mixminion._minionlib as _ml
+import time
from types import StringType
-from mixminion.Common import MixError, MixFatalError, info, warn, error, \
- log, debug
+
+import mixminion._minionlib as _ml
+from mixminion.Common import MixError, MixFatalError, getLog
from mixminion.Crypto import sha1
from mixminion.Packet import MESSAGE_LEN, DIGEST_LEN
__all__ = [ 'AsyncServer', 'ListenConnection', 'MMTPServerConnection',
'MMTPClientConnection' ]
-# Suppress trace-debugging
-def trace(x):
- #log("TRACE", x)
- pass
+trace = getLog().trace
+info = getLog().info
+debug = getLog().info
+warn = getLog().warn
+error = getLog().error
class AsyncServer:
"""AsyncServer is the core of a general-purpose asynchronous
@@ -75,7 +77,7 @@
return
else:
raise e
-
+
for fd in readfds:
trace("Got a read on "+str(fd))
self.readers[fd].handleRead()
@@ -189,6 +191,7 @@
method is called.
"""
# Fields:
+ # lastActivity: the last time when we had a read or a write.
# __con: an underlying TLS object
# __state: a callback to use whenever we get a read or a write. May
# throw _ml.TLSWantRead or _ml.TLSWantWrite.
@@ -342,6 +345,8 @@
reading or writing, or until the current __state becomes
None.
"""
+ self.lastActivity = time.time()
+
try:
# We have a while loop here so that, upon entering a new
# state, we immediately see if we can go anywhere with it
@@ -389,10 +394,10 @@
#----------------------------------------------------------------------
# XXXX Need to support future protos.
-PROTOCOL_STRING = "PROTOCOL 1.0\n"
-PROTOCOL_RE = re.compile("PROTOCOL ([^\s\r\n]+)\n")
-SEND_CONTROL = "SEND\n" #XXXX Not as in spec
-RECEIVED_CONTROL = "RECEIVED\n" #XXXX Not as in spec
+PROTOCOL_STRING = "PROTOCOL 1.0\r\n"
+PROTOCOL_RE = re.compile("PROTOCOL ([^\s\r\n]+)\r\n")
+SEND_CONTROL = "SEND\r\n" #XXXX Not as in spec
+RECEIVED_CONTROL = "RECEIVED\r\n" #XXXX Not as in spec
SEND_CONTROL_LEN = len(SEND_CONTROL)
RECEIVED_CONTROL_LEN = len(RECEIVED_CONTROL)
SEND_RECORD_LEN = len(SEND_CONTROL) + MESSAGE_LEN + DIGEST_LEN
@@ -471,6 +476,7 @@
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setblocking(0)
self.keyID = keyID
+ self.ip = ip
try:
sock.connect((ip, port))
except socket.error:
@@ -488,8 +494,14 @@
"""Called when we're done with the client side negotations.
Begins sending the protocol string.
"""
- peer_pk = self.getPeerPK()
- # Check this!
+ keyID = sha1(self.getPeerPK().encode_key(public=1))
+ if self.keyID is not None:
+ if keyID != self.keyID:
+ warn("Got unexpected Key ID from %s", self.ip)
+ self.shutdown(err=1)
+ else:
+ debug("KeyID is valid")
+
self.beginWrite(PROTOCOL_STRING)
self.finished = self.__sentProtocol
Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- Queue.py 5 Jul 2002 19:50:27 -0000 1.4
+++ Queue.py 9 Jul 2002 04:07:14 -0000 1.5
@@ -11,7 +11,7 @@
import time
import stat
-from mixminion.Common import MixError, MixFatalError, secureDelete, warn
+from mixminion.Common import MixError, MixFatalError, secureDelete, getLog
from mixminion.Crypto import AESCounterPRNG
__all__ = [ 'Queue' ]
@@ -65,7 +65,7 @@
self.dir = location
if not os.path.isabs(location):
- warn("Queue path %s isn't absolute.", location)
+ getLog().warn("Queue path %s isn't absolute.", location)
if os.path.exists(location) and not os.path.isdir(location):
raise MixFatalError("%s is not a directory" % location)
@@ -80,7 +80,7 @@
mode = os.stat(location)[stat.ST_MODE]
if mode & 0077:
# FFFF be more Draconian.
- warn("Worrisome more %o on directory %s", mode, location)
+ getLog().warn("Worrisome more %o on directory %s", mode, location)
if scrub:
self.cleanQueue(1)
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- test.py 5 Jul 2002 23:34:32 -0000 1.10
+++ test.py 9 Jul 2002 04:07:14 -0000 1.11
@@ -18,8 +18,10 @@
import threading
import time
import atexit
+import tempfile
+import types
-from mixminion.Common import MixError, MixFatalError
+from mixminion.Common import MixError, MixFatalError, getLog
try:
import unittest
@@ -38,17 +40,31 @@
r.append(chr(c))
return "".join(r)
-def try_unlink(fname):
- try:
- os.unlink(fname)
- except OSError:
- pass
+def try_unlink(fnames):
+ if isinstance(fnames, types.StringType):
+ fnames = [fnames]
+ for fname in fnames:
+ try:
+ os.unlink(fname)
+ except OSError:
+ pass
def try_unlink_db(fname):
'''Try to unlink an anydbm file(s)'''
for suffix in ("", ".bak", ".dat", ".dir"):
try_unlink(fname+suffix)
+_unlink_on_exit_list = []
+
+def unlink_db_on_exit(fname):
+ for suffix in ("", ".bak", ".dat", ".dir"):
+ _unlink_on_exit_list.append(fname+suffix)
+
+def unlink_on_exit(*files):
+ _unlink_on_exit_list.extend(files)
+
+atexit.register(try_unlink, _unlink_on_exit_list)
+
#----------------------------------------------------------------------
import mixminion._minionlib as _ml
@@ -227,6 +243,29 @@
self.failUnlessRaises(TypeError, p3.crypt, msg1, 0, 0)
self.failUnlessRaises(TypeError, p2.encode_key, 0)
self.failUnlessRaises(TypeError, p3.encode_key, 0)
+
+ tf = tempfile.mktemp()
+ tf_pub = tf + "1"
+ tf_prv = tf + "2"
+ tf_enc = tf + "3"
+ unlink_on_exit(tf_pub, tf_prv, tf_enc)
+
+ p.PEM_write_key(open(tf_pub,'w'), 1)
+ p.PEM_write_key(open(tf_prv,'w'), 0)
+ p.PEM_write_key(open(tf_enc,'w'), 0, "top sekrit")
+ p2 = _ml.rsa_PEM_read_key(open(tf_pub, 'r'), 1)
+ self.assertEquals(p.get_public_key(), p2.get_public_key())
+
+ p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0)
+ self.assertEquals(p.encode_key(0), p2.encode_key(0))
+
+ self.failUnlessRaises(_ml.CryptoError,
+ _ml.rsa_PEM_read_key,
+ open(tf_enc, 'r'), 0)
+
+ p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0, "top sekrit")
+ self.assertEquals(p.encode_key(0), p2.encode_key(0))
+
#----------------------------------------------------------------------
import mixminion.Crypto
from mixminion.Crypto import *
@@ -503,14 +542,9 @@
class HashLogTests(unittest.TestCase):
def test_hashlog(self):
- import tempfile
fname = tempfile.mktemp(".db")
- try:
- self.hashlogTestImpl(fname)
- finally:
- try_unlink_db(fname)
+ unlink_db_on_exit(fname)
- def hashlogTestImpl(self,fname):
h = [HashLog(fname, "Xyzzy")]
notseen = lambda hash,self=self,h=h: self.assert_(not h[0].seenHash(hash))
@@ -927,6 +961,7 @@
self.pk2 = BMTSupport.pk2
self.pk3 = BMTSupport.pk3
self.tmpfile = mktemp(".db")
+ unlink_db_on_exit(self.tmpfile)
h = self.hlog = HashLog(self.tmpfile, "Z"*20)
n_1 = pk_get_modulus(self.pk1)
n_2 = pk_get_modulus(self.pk2)
@@ -941,7 +976,6 @@
def tearDown(self):
self.hlog.close()
- try_unlink_db(self.tmpfile)
def do_test_chain(self, m, sps, routingtypes, routinginfo, payload,
appkey=None):
@@ -1142,7 +1176,7 @@
from mixminion.Queue import Queue
def removeTempDirs(*dirs):
- print "Removing temporary dirs"
+ print "Waiting for shred processes to finish."
waitForChildren()
for d in dirs:
if os.path.isdir(d):
@@ -1281,14 +1315,33 @@
TEST_PORT = 40102
+dhfile = pkfile = certfile = None
+
def _getTLSContext(isServer):
+ global dhfile
+ global pkfile
+ global certfile
if isServer:
- d = "/home/nickm/src/ssl_sandbox/"
- for f in (d+"server.cert",d+"server.pk",d+"dh"):
- assert os.path.exists(f)
- #XXXX Generate these if they don't exist; look in a saner place.
- pk = _ml.rsa_PEM_read_key(open(d+"server.pk", 'r'), 0)
- return _ml.TLSContext_new(d+"server.cert",pk,d+"dh")
+ if dhfile is None:
+ f = tempfile.mktemp()
+ dhfile = f+"_dh"
+ pkfile = f+"_pk"
+ certfile = f+"_cert"
+ dh_fname = os.environ.get("MM_TEST_DHPARAMS", None)
+ if dh_fname:
+ dhfile = dh_fname
+ if not os.path.exists(dh_fname):
+ _ml.generate_dh_parameters(dhfile, 0)
+ else:
+ _ml.generate_dh_parameters(dhfile, 0)
+ unlink_on_exit(dhfile)
+ pk = _ml.rsa_generate(1024, 65535)
+ pk.PEM_write_key(open(pkfile, 'w'), 0)
+ _ml.generate_cert(certfile, pk, 365, "Testing certificate")
+ unlink_on_exit(certfile, pkfile)
+
+ pk = _ml.rsa_PEM_read_key(open(pkfile, 'r'), 0)
+ return _ml.TLSContext_new(certfile, pk, dhfile)
else:
return _ml.TLSContext_new()
@@ -1306,7 +1359,10 @@
listener = mixminion.MMTPServer.ListenConnection("127.0.0.1",
TEST_PORT, 5, conFactory)
listener.register(server)
- return server, listener, messagesIn
+ pk = _ml.rsa_PEM_read_key(open(pkfile, 'r'), public=0)
+ keyid = sha1(pk.encode_key(1))
+
+ return server, listener, messagesIn, keyid
class MMTPTests(unittest.TestCase):
@@ -1331,7 +1387,7 @@
self.doTest(self._testNonblockingTransmission)
def _testBlockingTransmission(self):
- server, listener, messagesIn = _getMMTPServer()
+ server, listener, messagesIn, keyid = _getMMTPServer()
self.listener = listener
self.server = server
@@ -1340,7 +1396,7 @@
server.process(0.1)
t = threading.Thread(None,
mixminion.MMTPClient.sendMessages,
- args=("127.0.0.1", TEST_PORT, None, messages))
+ args=("127.0.0.1", TEST_PORT, keyid, messages))
t.start()
while len(messagesIn) < 2:
server.process(0.1)
@@ -1352,14 +1408,14 @@
self.failUnless(messagesIn == messages)
def _testNonblockingTransmission(self):
- server, listener, messagesIn = _getMMTPServer()
+ server, listener, messagesIn, keyid = _getMMTPServer()
self.listener = listener
self.server = server
messages = ["helloxxx"*4096, "helloyyy"*4096]
async = mixminion.MMTPServer.AsyncServer()
clientcon = mixminion.MMTPServer.MMTPClientConnection(
- _getTLSContext(0), "127.0.0.1", TEST_PORT, None, messages[:], None)
+ _getTLSContext(0), "127.0.0.1", TEST_PORT, keyid, messages[:], None)
clientcon.register(async)
def clientThread(clientcon=clientcon, async=async):
while not clientcon.isShutdown():
@@ -1384,6 +1440,7 @@
suite = unittest.TestSuite()
loader = unittest.TestLoader()
tc = loader.loadTestsFromTestCase
+ getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
suite.addTest(tc(MinionlibCryptoTests))
suite.addTest(tc(CryptoTests))
suite.addTest(tc(FormatTests))
@@ -1391,14 +1448,7 @@
suite.addTest(tc(BuildMessageTests))
suite.addTest(tc(PacketHandlerTests))
suite.addTest(tc(QueueTests))
-
- # XXXX This test won't work for anybody but me until I get DH/keygen
- # XXXX working happily. -NM
- if os.path.exists("/home/nickm/src/ssl_sandbox/dh"):
- print "Including mmtp tests XXXX"
- suite.addTest(tc(MMTPTests))
- else:
- print "excluding mmtp tests XXXX"
+ suite.addTest(tc(MMTPTests))
return suite
def testAll():
@@ -1406,18 +1456,4 @@
if __name__ == '__main__':
init_crypto()
-
- d = "/home/nickm/src/ssl_sandbox/"
-
-## print "dh"
-## _ml.generate_dh_parameters(d+"dh", 0)
-## print "rsa"
-## pk = _ml.rsa_generate(1024, 65535)
-## pk.PEM_write_key(open(d+"server.pk", 'w'),0)
-## print "cert"
-## _ml.generate_cert(d+"server.cert", pk, 365, "foobar")
-## print "go!"
-
-## print "-----------"
-
testAll()