[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[minion-cvs] Made TODO reflect open spec issues; updates to reflect ...



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv12628/lib/mixminion

Modified Files:
	Common.py HashLog.py MMTPClient.py MMTPServer.py Queue.py 
	test.py 
Added Files:
	README 
Log Message:
Made TODO reflect open spec issues; updates to reflect spec; more testing.


TODO file: remove completed items; add a list of what's still
blocking; push advanced SSL stuff off till release "0.2".

src/README, lib/mixminion/README: describe the purpose of each file.

Common.py, HashLog.py, Queue.py: refactor logging a bit

MMTPClient.py: Check hash of server's key.  Change from LF to CRLF.

MMTPServer.py: 
	- Remember when each connection last saw action.  
	- Check hash of server's key.
	- Change from LF to CRLF.

test.py:
	- Add tests for PEM-encoded RSA keys.
	- Made all the tests absolutely clean up /tmp after themselves.
	- Added support for automatic generation of private keys and DH
	  parameters for server testing.  The tests for MMTP can now run
	  on a machine other than my own.
	- Test keyid checking
	- Suppress log messages during testing.



--- NEW FILE: README ---
DIRECTORY: lib/mixminion/

Python implementation of the Mixminion anonymizing system.

__init__.py
	Required by Python to make this group of files a package.

Crypto.py
	Wrappers for low-level cryptographic functionality

Common.py
	Helper routines for OS functionality, base exceptions, etc.

HashLog.py
	Class to keep a record of packets that we've processed.

Modules.py
	Registry of delievery modules, and implementations of the most
	simple ones

Queue.py 
	A directory-based unordered message queue.

Packet.py
	Data structures from mixminion spec, and routines to serialize
	and unserialize them.

ServerInfo.py
	Representation for a server's descriptor block

Config.py
	Configuration of the local server.

BuildMessage.py
	Functions to construct packets and reply blocks

PacketHandler.py
	Class to unwrap and process packets.

MMTPClient.py
	Blocking client-side implementation of the Mixminion Transfer protocol

MMTPServer.py
	Nonblocking implementation of MMTP

benchmark.py
test.py
	Unit/performance tests


Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- Common.py	5 Jul 2002 19:50:27 -0000	1.7
+++ Common.py	9 Jul 2002 04:07:14 -0000	1.8
@@ -7,8 +7,7 @@
 
 __all__ = [ 'MixError', 'MixFatalError', 'onReset', 'onTerminate',
             'installSignalHandlers', 'secureDelete', 'secureRename',
-            'ceilDiv', 'floorDiv', 'debug', 'warn', 'info', 'error',
-            'fatal' ]
+            'ceilDiv', 'floorDiv', 'getLog' ]
 
 import os
 import signal
@@ -129,14 +128,29 @@
     def write(self, severity, message):
         print >> self.file, "%s [%s] %s" % (_logtime(), severity, message)
 
+
+_SEVERITIES = { 'TRACE' : -2,
+                'DEBUG' : -1,
+                'INFO' : 0,
+                'WARN' : 1,
+                'ERROR': 2,
+                'FATAL' : 3 }
+
 class Log:
     def __init__(self, minSeverity):
         self.handlers = []
+        self.setMinSeverity(minSeverity)
+        onReset(self.reset)
+        onTerminate(self.close)
+
+    def setMinSeverity(self, minSeverity):
+        self.severity = _SEVERITIES.get(minSeverity, 1)
 
     def addHandler(self, handler):
         self.handlers.append(handler)
 
     def reset(self):
+        # FFFF reload configuration information here?
         for h in self.handlers:
             h.reset()
 
@@ -145,12 +159,15 @@
             h.close()
         
     def log(self, severity, message, *args):
-        # Check that severity is okay.
+        if _SEVERITIES.get(severity, 100) < self.severity:
+            return
+        m = message % args
         for h in self.handlers:
-            h.write(severity, message % args)
+            h.write(severity, m)
 
+    def trace(self, message, *args):
+        self.log("TRACE", message, *args)
     def debug(self, message, *args):
-        # Need a means to filter messages
         self.log("DEBUG", message, *args)
     def info(self, message, *args):
         self.log("INFO", message, *args)
@@ -161,16 +178,19 @@
     def fatal(self, message, *args):
         self.log("FATAL", message, *args)
 
-_theLog = Log('DEBUG')
-_theLog.addHandler(ConsoleLogTarget(sys.stderr))
-# XXXX Configure the log for real
+_theLog = None
+
+def getLog():
+    """Return the MixMinion log object."""
+    global _theLog
+    if _theLog is None:
+        # XXXX Configure the log for real
+        _theLog = Log('DEBUG')
+        _theLog.addHandler(ConsoleLogTarget(sys.stderr))
+        
+    return _theLog
+
 
-log = _theLog.log
-debug = _theLog.debug
-info = _theLog.info
-warn = _theLog.warn
-error = _theLog.error
-fatal = _theLog.fatal
 
 #----------------------------------------------------------------------
 # Signal handling
@@ -191,7 +211,6 @@
     """Given a 0-argument function fn, cause fn to be invoked when
        this process next receives a SIGTERM."""
     terminateHooks.append(fn)
-
 
 def waitForChildren():
     """Wait until all subprocesses have finished.  Useful for testing.""" 

Index: HashLog.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/HashLog.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- HashLog.py	5 Jul 2002 19:50:27 -0000	1.8
+++ HashLog.py	9 Jul 2002 04:07:14 -0000	1.9
@@ -6,7 +6,7 @@
    Persistant memory for the hashed secrets we've seen."""
 
 import anydbm, dumbdbm
-from mixminion.Common import MixFatalError, warn
+from mixminion.Common import MixFatalError, getLog
 
 __all__ = [ 'HashLog' ]
 
@@ -39,7 +39,7 @@
            'keyid'."""
         self.log = anydbm.open(filename, 'c')
         if isinstance(self.log, dumbdbm._Database):
-            warn("Warning: logging packet digests to a flat file.")
+            getLog().warn("Warning: logging packet digests to a flat file.")
         try:
             if self.log["KEYID"] != keyid:
                 raise MixFatalError("Log KEYID does not match current KEYID")

Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- MMTPClient.py	1 Jul 2002 18:03:05 -0000	1.3
+++ MMTPClient.py	9 Jul 2002 04:07:14 -0000	1.4
@@ -44,25 +44,28 @@
         #XXXX session resumption
         self.tls.connect()
         peer_pk = self.tls.get_peer_cert_pk()
-        # XXXX Check the key ... how exactly is this to be hashed?
+        keyID = sha1(self.tls.get_peer_cert_pk().encode_key(public=1))
+        if self.targetKeyID is not None and (keyID != self.targetKeyID):
+            raise MixProtocolError("Bad Key ID: Expected %r but got %r" % (
+                self.targetKeyID, keyID))
         
         ####
         # Protocol negotiation
         # For now, we only support 1.0
-        self.tls.write("PROTOCOL 1.0\n")
-        inp = self.tls.read(len("PROTOCOL 1.0\n"))
-        if inp != "PROTOCOL 1.0\n":
+        self.tls.write("PROTOCOL 1.0\r\n")
+        inp = self.tls.read(len("PROTOCOL 1.0\r\n"))
+        if inp != "PROTOCOL 1.0\r\n":
             raise MixProtocolError("Protocol negotiation failed")
         
     def sendPacket(self, packet):
         """Send a single packet to a server."""
         assert len(packet) == 1<<15
-        self.tls.write("SEND\n")
+        self.tls.write("SEND\r\n")
         self.tls.write(packet)
         self.tls.write(sha1(packet+"SEND"))
         
-        inp = self.tls.read(len("RECEIVED\n")+20)
-        if inp != "RECEIVED\n"+sha1(packet+"RECEIVED"):
+        inp = self.tls.read(len("RECEIVED\r\n")+20)
+        if inp != "RECEIVED\r\n"+sha1(packet+"RECEIVED"):
             raise MixProtocolError("Bad ACK received")
 
     def shutdown(self):

Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- MMTPServer.py	5 Jul 2002 19:50:27 -0000	1.5
+++ MMTPServer.py	9 Jul 2002 04:07:14 -0000	1.6
@@ -16,31 +16,33 @@
 
 # NOTE FOR THE CURIOUS: The 'asyncore' module in the standard library
 #    is another general select/poll wrapper... so why are we using our
-#    own?  Basically, because asyncore has IMO a couple of misdesigns,
-#    the largest of which is that it has the 'server loop' periodically
-#    query the connections for their status, whereas we have the
-#    connections inform the server of their status whenever they
-#    change.  This latter approach turns out to be far easier to use
-#    with TLS.
+#    own?  Basically, because asyncore has IMO a couple of mismatches
+#    with our design, the largest of which is that it has the 'server
+#    loop' periodically query the connections for their status,
+#    whereas we have the connections inform the server of their status
+#    whenever they change.  This latter approach turns out to be far
+#    easier to use with TLS.
 
 import errno
 import socket
 import select
 import re
-import mixminion._minionlib as _ml
+import time
 from types import StringType
-from mixminion.Common import MixError, MixFatalError, info, warn, error, \
-     log, debug
+
+import mixminion._minionlib as _ml
+from mixminion.Common import MixError, MixFatalError, getLog
 from mixminion.Crypto import sha1
 from mixminion.Packet import MESSAGE_LEN, DIGEST_LEN
 
 __all__ = [ 'AsyncServer', 'ListenConnection', 'MMTPServerConnection',
             'MMTPClientConnection' ]
 
-# Suppress trace-debugging
-def trace(x):
-    #log("TRACE", x)
-    pass
+trace = getLog().trace
+info = getLog().info
+debug = getLog().info
+warn = getLog().warn
+error = getLog().error
 
 class AsyncServer:
     """AsyncServer is the core of a general-purpose asynchronous
@@ -75,7 +77,7 @@
                 return
             else:
                 raise e
-        
+
         for fd in readfds:
             trace("Got a read on "+str(fd))
             self.readers[fd].handleRead()
@@ -189,6 +191,7 @@
        method is called.
     """
     # Fields:
+    #    lastActivity: the last time when we had a read or a write.
     #    __con: an underlying TLS object
     #    __state: a callback to use whenever we get a read or a write. May
     #           throw _ml.TLSWantRead or _ml.TLSWantWrite.
@@ -342,6 +345,8 @@
            reading or writing, or until the current __state becomes
            None.
         """
+        self.lastActivity = time.time()
+        
         try:
             # We have a while loop here so that, upon entering a new
             # state, we immediately see if we can go anywhere with it
@@ -389,10 +394,10 @@
     
 #----------------------------------------------------------------------
 # XXXX Need to support future protos.
-PROTOCOL_STRING      = "PROTOCOL 1.0\n"
-PROTOCOL_RE = re.compile("PROTOCOL ([^\s\r\n]+)\n")
-SEND_CONTROL         = "SEND\n" #XXXX Not as in spec
-RECEIVED_CONTROL     = "RECEIVED\n" #XXXX Not as in spec
+PROTOCOL_STRING      = "PROTOCOL 1.0\r\n"
+PROTOCOL_RE = re.compile("PROTOCOL ([^\s\r\n]+)\r\n")
+SEND_CONTROL         = "SEND\r\n" #XXXX Not as in spec
+RECEIVED_CONTROL     = "RECEIVED\r\n" #XXXX Not as in spec
 SEND_CONTROL_LEN     = len(SEND_CONTROL)
 RECEIVED_CONTROL_LEN = len(RECEIVED_CONTROL)
 SEND_RECORD_LEN      = len(SEND_CONTROL) + MESSAGE_LEN + DIGEST_LEN
@@ -471,6 +476,7 @@
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.setblocking(0)
         self.keyID = keyID
+        self.ip = ip
         try:
             sock.connect((ip, port))
         except socket.error:
@@ -488,8 +494,14 @@
         """Called when we're done with the client side negotations.
            Begins sending the protocol string.
         """
-        peer_pk = self.getPeerPK()
-        # Check this!
+        keyID = sha1(self.getPeerPK().encode_key(public=1))
+        if self.keyID is not None:
+            if keyID != self.keyID:
+                warn("Got unexpected Key ID from %s", self.ip)
+                self.shutdown(err=1)
+            else:
+                debug("KeyID is valid")
+
         self.beginWrite(PROTOCOL_STRING)
         self.finished = self.__sentProtocol
 

Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- Queue.py	5 Jul 2002 19:50:27 -0000	1.4
+++ Queue.py	9 Jul 2002 04:07:14 -0000	1.5
@@ -11,7 +11,7 @@
 import time
 import stat
 
-from mixminion.Common import MixError, MixFatalError, secureDelete, warn
+from mixminion.Common import MixError, MixFatalError, secureDelete, getLog
 from mixminion.Crypto import AESCounterPRNG
 
 __all__ = [ 'Queue' ]
@@ -65,7 +65,7 @@
         self.dir = location
 
         if not os.path.isabs(location):
-            warn("Queue path %s isn't absolute.", location)
+            getLog().warn("Queue path %s isn't absolute.", location)
 
         if os.path.exists(location) and not os.path.isdir(location):
             raise MixFatalError("%s is not a directory" % location)
@@ -80,7 +80,7 @@
         mode = os.stat(location)[stat.ST_MODE]
         if mode & 0077:
             # FFFF be more Draconian.
-            warn("Worrisome more %o on directory %s", mode, location)
+            getLog().warn("Worrisome more %o on directory %s", mode, location)
 
         if scrub:
             self.cleanQueue(1)

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- test.py	5 Jul 2002 23:34:32 -0000	1.10
+++ test.py	9 Jul 2002 04:07:14 -0000	1.11
@@ -18,8 +18,10 @@
 import threading
 import time
 import atexit
+import tempfile
+import types
 
-from mixminion.Common import MixError, MixFatalError
+from mixminion.Common import MixError, MixFatalError, getLog
 
 try:
     import unittest
@@ -38,17 +40,31 @@
         r.append(chr(c))
     return "".join(r)
 
-def try_unlink(fname):
-    try:
-        os.unlink(fname)
-    except OSError:
-        pass
+def try_unlink(fnames):
+    if isinstance(fnames, types.StringType):
+        fnames = [fnames]
+    for fname in fnames:
+        try:
+            os.unlink(fname)
+        except OSError:
+            pass
 
 def try_unlink_db(fname):
     '''Try to unlink an anydbm file(s)'''
     for suffix in ("", ".bak", ".dat", ".dir"):
         try_unlink(fname+suffix)
 
+_unlink_on_exit_list = []
+
+def unlink_db_on_exit(fname):
+    for suffix in ("", ".bak", ".dat", ".dir"):
+        _unlink_on_exit_list.append(fname+suffix)
+    
+def unlink_on_exit(*files):
+    _unlink_on_exit_list.extend(files)
+
+atexit.register(try_unlink, _unlink_on_exit_list)    
+
 #----------------------------------------------------------------------
 import mixminion._minionlib as _ml
 
@@ -227,6 +243,29 @@
         self.failUnlessRaises(TypeError, p3.crypt, msg1, 0, 0)
         self.failUnlessRaises(TypeError, p2.encode_key, 0)
         self.failUnlessRaises(TypeError, p3.encode_key, 0)
+
+        tf = tempfile.mktemp()
+        tf_pub = tf + "1"
+        tf_prv = tf + "2"
+        tf_enc = tf + "3"
+        unlink_on_exit(tf_pub, tf_prv, tf_enc)
+
+        p.PEM_write_key(open(tf_pub,'w'), 1)
+        p.PEM_write_key(open(tf_prv,'w'), 0)
+        p.PEM_write_key(open(tf_enc,'w'), 0, "top sekrit")
+        p2 = _ml.rsa_PEM_read_key(open(tf_pub, 'r'), 1)
+        self.assertEquals(p.get_public_key(), p2.get_public_key())
+        
+        p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0)
+        self.assertEquals(p.encode_key(0), p2.encode_key(0))
+
+        self.failUnlessRaises(_ml.CryptoError,
+                              _ml.rsa_PEM_read_key,
+                              open(tf_enc, 'r'), 0)
+                                  
+        p2 = _ml.rsa_PEM_read_key(open(tf_prv, 'r'), 0, "top sekrit")
+        self.assertEquals(p.encode_key(0), p2.encode_key(0))
+            
 #----------------------------------------------------------------------
 import mixminion.Crypto
 from mixminion.Crypto import *
@@ -503,14 +542,9 @@
 
 class HashLogTests(unittest.TestCase):
     def test_hashlog(self):
-        import tempfile
         fname = tempfile.mktemp(".db")
-        try:
-            self.hashlogTestImpl(fname)
-        finally:
-            try_unlink_db(fname)
+        unlink_db_on_exit(fname)
 
-    def hashlogTestImpl(self,fname):
         h = [HashLog(fname, "Xyzzy")]
 
         notseen = lambda hash,self=self,h=h: self.assert_(not h[0].seenHash(hash))
@@ -927,6 +961,7 @@
         self.pk2 = BMTSupport.pk2
         self.pk3 = BMTSupport.pk3
         self.tmpfile = mktemp(".db")
+        unlink_db_on_exit(self.tmpfile)
         h = self.hlog = HashLog(self.tmpfile, "Z"*20)
         n_1 = pk_get_modulus(self.pk1)
         n_2 = pk_get_modulus(self.pk2)
@@ -941,7 +976,6 @@
 
     def tearDown(self):
         self.hlog.close()
-        try_unlink_db(self.tmpfile)
 
     def do_test_chain(self, m, sps, routingtypes, routinginfo, payload,
                       appkey=None):
@@ -1142,7 +1176,7 @@
 from mixminion.Queue import Queue
 
 def removeTempDirs(*dirs):
-    print "Removing temporary dirs"
+    print "Waiting for shred processes to finish."
     waitForChildren()
     for d in dirs:
         if os.path.isdir(d):
@@ -1281,14 +1315,33 @@
 
 TEST_PORT = 40102
 
+dhfile = pkfile = certfile = None
+
 def _getTLSContext(isServer):
+    global dhfile
+    global pkfile
+    global certfile
     if isServer:
-        d = "/home/nickm/src/ssl_sandbox/"
-        for f in (d+"server.cert",d+"server.pk",d+"dh"):
-            assert os.path.exists(f)
-        #XXXX Generate these if they don't exist; look in a saner place.
-	pk = _ml.rsa_PEM_read_key(open(d+"server.pk", 'r'), 0)
-        return _ml.TLSContext_new(d+"server.cert",pk,d+"dh")
+        if dhfile is None:
+            f = tempfile.mktemp()
+            dhfile = f+"_dh"
+            pkfile = f+"_pk"
+            certfile = f+"_cert"
+            dh_fname = os.environ.get("MM_TEST_DHPARAMS", None)
+            if dh_fname:
+                dhfile = dh_fname
+                if not os.path.exists(dh_fname):
+                    _ml.generate_dh_parameters(dhfile, 0)
+            else:
+                _ml.generate_dh_parameters(dhfile, 0)
+                unlink_on_exit(dhfile)
+            pk = _ml.rsa_generate(1024, 65535)
+            pk.PEM_write_key(open(pkfile, 'w'), 0)
+            _ml.generate_cert(certfile, pk, 365, "Testing certificate")
+            unlink_on_exit(certfile, pkfile)
+            
+	pk = _ml.rsa_PEM_read_key(open(pkfile, 'r'), 0)
+        return _ml.TLSContext_new(certfile, pk, dhfile)
     else:
         return _ml.TLSContext_new()
 
@@ -1306,7 +1359,10 @@
         listener = mixminion.MMTPServer.ListenConnection("127.0.0.1",
                                                      TEST_PORT, 5, conFactory)
         listener.register(server)
-        return server, listener, messagesIn
+        pk = _ml.rsa_PEM_read_key(open(pkfile, 'r'), public=0)
+        keyid = sha1(pk.encode_key(1))
+        
+        return server, listener, messagesIn, keyid
 
 class MMTPTests(unittest.TestCase):
 
@@ -1331,7 +1387,7 @@
         self.doTest(self._testNonblockingTransmission)
     
     def _testBlockingTransmission(self):
-        server, listener, messagesIn = _getMMTPServer()
+        server, listener, messagesIn, keyid = _getMMTPServer()
         self.listener = listener
         self.server = server
         
@@ -1340,7 +1396,7 @@
         server.process(0.1)
         t = threading.Thread(None,
                              mixminion.MMTPClient.sendMessages,
-                             args=("127.0.0.1", TEST_PORT, None, messages))
+                             args=("127.0.0.1", TEST_PORT, keyid, messages))
         t.start()
         while len(messagesIn) < 2:
             server.process(0.1)
@@ -1352,14 +1408,14 @@
         self.failUnless(messagesIn == messages)
 
     def _testNonblockingTransmission(self):
-        server, listener, messagesIn = _getMMTPServer()
+        server, listener, messagesIn, keyid = _getMMTPServer()
         self.listener = listener
         self.server = server
 
         messages = ["helloxxx"*4096, "helloyyy"*4096]
         async = mixminion.MMTPServer.AsyncServer()
         clientcon = mixminion.MMTPServer.MMTPClientConnection(
-            _getTLSContext(0), "127.0.0.1", TEST_PORT, None, messages[:], None)
+           _getTLSContext(0), "127.0.0.1", TEST_PORT, keyid, messages[:], None)
         clientcon.register(async)
         def clientThread(clientcon=clientcon, async=async):
             while not clientcon.isShutdown():
@@ -1384,6 +1440,7 @@
     suite = unittest.TestSuite()
     loader = unittest.TestLoader()
     tc = loader.loadTestsFromTestCase
+    getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
     suite.addTest(tc(MinionlibCryptoTests))
     suite.addTest(tc(CryptoTests))
     suite.addTest(tc(FormatTests))
@@ -1391,14 +1448,7 @@
     suite.addTest(tc(BuildMessageTests))
     suite.addTest(tc(PacketHandlerTests))
     suite.addTest(tc(QueueTests))
-
-    # XXXX This test won't work for anybody but me until I get DH/keygen
-    # XXXX working happily. -NM
-    if os.path.exists("/home/nickm/src/ssl_sandbox/dh"):
-	print "Including mmtp tests XXXX"
-        suite.addTest(tc(MMTPTests))
-    else:
-	print "excluding mmtp tests XXXX"
+    suite.addTest(tc(MMTPTests))
     return suite
 
 def testAll():
@@ -1406,18 +1456,4 @@
 
 if __name__ == '__main__':
     init_crypto()
-
-    d = "/home/nickm/src/ssl_sandbox/"
-
-##      print "dh"
-##      _ml.generate_dh_parameters(d+"dh", 0)
-##      print "rsa"
-##      pk = _ml.rsa_generate(1024, 65535)
-##      pk.PEM_write_key(open(d+"server.pk", 'w'),0)
-##      print "cert"
-##      _ml.generate_cert(d+"server.cert", pk, 365, "foobar")
-##      print "go!"
-    
-##      print "-----------"
-
     testAll()