[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[minion-cvs] Refactor and reformat key rings; other bugfixes and UI ...



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv23020/lib/mixminion

Modified Files:
	BuildMessage.py ClientDirectory.py ClientMain.py 
	ClientUtils.py test.py 
Log Message:
Refactor and reformat key rings; other bugfixes and UI improvements;
add tests.

BuildMessage: 
  - Remove lots of dead code; change userKeys format to allow SURB key
    rotation.

ClientDirectory:
  - Don't check message side when delivering fragments to the recipients.

ClientMain:
  - Refactor ClientKeyring to use new Keyring class in ClientUtil.
  - Debug and name --deliver-fragments
  - Let users know when we're creating a configuration file for them.
  - Refactor inspectQueue so that all CLI-specific stuff happens in ClientMain

ClientUtils:
  - Completely change keyring format to match specification
     - Say 'passphrase' instead of 'password'
     - Since keyrings are the only users of our 'encryptedFoo' functionality,
       don't export anything besides a Keyring class.
     - Document everything and add support for backward compatibility
     - Add SURB key rotation.
  - Refactor inspectQueue so that all CLI-specific stuff happens in ClientMain

test:
  - Remove a few long-dead tests.
  - Check new nickname restrictions
  - Test new keyring formats and behavior.

  



Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.61
retrieving revision 1.62
diff -u -d -r1.61 -r1.62
--- BuildMessage.py	10 Nov 2003 04:12:20 -0000	1.61
+++ BuildMessage.py	20 Nov 2003 08:51:27 -0000	1.62
@@ -340,16 +340,18 @@
 
            key: an RSA key to decode encrypted forward messages, or None
            userKeys: a map from identity names to keys for reply blocks,
-                or None.
+                or None. DOCDOC : prefer list of (name,key)
 
        If we can successfully decrypt the payload, we return it.  If we
        might be able to decrypt the payload given more/different keys,
        we return None.  If the payload is corrupt, we raise MixError.
     """
     if userKeys is None:
-        userKeys = {}
+        userKeys = []
     elif type(userKeys) is types.StringType:
-        userKeys = { "" : userKeys }
+        userKeys = [ ("", userKeys) ]
+    elif type(userKeys) is types.DictType:
+        userKeys = userKeys.items()
 
     if len(payload) != PAYLOAD_LEN:
         raise MixError("Wrong payload length")
@@ -368,16 +370,15 @@
     # If H(tag|userKey|"Validate") ends with 0, then the message _might_
     # be a reply message using H(tag|userKey|"Generate") as the seed for
     # its master secrets.  (There's a 1-in-256 chance that it isn't.)
-    if userKeys:
-        for name,userKey in userKeys.items():
-            if Crypto.sha1(tag+userKey+"Validate")[-1] == '\x00':
-                try:
-                    p = _decodeStatelessReplyPayload(payload, tag, userKey)
-                    if name:
-                        LOG.info("Decoded reply message to identity %r", name)
-                    return p
-                except MixError:
-                    pass
+    for name,userKey in userKeys:
+        if Crypto.sha1(tag+userKey+"Validate")[-1] == '\x00':
+            try:
+                p = _decodeStatelessReplyPayload(payload, tag, userKey)
+                if name:
+                    LOG.info("Decoded reply message to identity %r", name)
+                return p
+            except MixError:
+                pass
 
     # If we have an RSA key, and none of the above steps get us a good
     # payload, then we may as well try to decrypt the start of tag+key with

Index: ClientDirectory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientDirectory.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- ClientDirectory.py	19 Nov 2003 09:48:09 -0000	1.15
+++ ClientDirectory.py	20 Nov 2003 08:51:27 -0000	1.16
@@ -1036,6 +1036,8 @@
                 sware.startswith("Mixminion 0.0.5alpha1")):
                 raise UIError("Server %s is running old software that doesn't support exit headers.", nickname)
 
+        exitKB = ceilDiv(self.exitSize, 1024)
+
         if self.isSSFragmented:
             dfsec = desc['Delivery/Fragmented']
             if not dfsec.get("Version"):
@@ -1044,24 +1046,31 @@
             if self.nFragments > dfsec.get("Maximum-Fragments",0):
                 raise UIError("Too many fragments for server %s to reassemble."
                               %nickname)
+        else:
+            # If we're not asking the server to defrag, we only need 32K
+            if exitKB > 32:
+                exitKB = 32
+
+        needFrom = self.headers.has_key("FROM")
+        
         if self.exitType in ('smtp', SMTP_TYPE):
             ssec = desc['Delivery/SMTP']
             if not ssec.get("Version"):
                 raise UIError("Server %s doesn't support SMTP"%nickname)
-            if self.headers.has_key("FROM") and not ssec['Allow-From']:
+            if needFrom and not ssec['Allow-From']:
                 raise UIError("Server %s doesn't support user-supplied From"%
                               nickname)
-            if floorDiv(self.exitSize,1024) > ssec['Maximum-Size']:
+            if exitKB > ssec['Maximum-Size']:
                 raise UIError("Message to long for server %s to deliver."%
                               nickname)
         elif self.exitType in ('mbox', MBOX_TYPE):
             msec = desc['Delivery/MBOX']
             if not msec.get("Version"):
                 raise UIError("Server %s doesn't support MBOX"%nickname)
-            if self.headers.has_key("FROM") and not msec['Allow-From']:
+            if needFrom and not msec['Allow-From']:
                 raise UIError("Server %s doesn't support user-supplied From"%
                               nickname)
-            if floorDiv(self.exitSize,1024) > msec['Maximum-Size']:
+            if exitKB > msec['Maximum-Size']:
                 raise UIError("Message to long for server %s to deliver."%
                               nickname)
         elif self.exitType in ('drop', DROP_TYPE):

Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.130
retrieving revision 1.131
diff -u -d -r1.130 -r1.131
--- ClientMain.py	19 Nov 2003 09:48:09 -0000	1.130
+++ ClientMain.py	20 Nov 2003 08:51:27 -0000	1.131
@@ -25,7 +25,7 @@
 from mixminion.Common import LOG, Lockfile, LockfileLocked, MixError, \
      MixFatalError, MixProtocolBadAuth, MixProtocolError, UIError, \
      UsageError, createPrivateDir, isPrintingAscii, isSMTPMailbox, readFile, \
-     stringContains, succeedingMidnight, writeFile, previousMidnight
+     stringContains, succeedingMidnight, writeFile, previousMidnight, floorDiv
 from mixminion.Packet import encodeMailHeaders, ParseError, parseMBOXInfo, \
      parseReplyBlocks, parseSMTPInfo, parseTextEncodedMessages, \
      parseTextReplyBlocks, ReplyBlock, MBOX_TYPE, SMTP_TYPE, DROP_TYPE, \
@@ -63,26 +63,29 @@
     """
     # XXXX Most of this class should go into ClientUtils?
     # XXXX006 Are the error messages here still reasonable?
+    # DOCDOC -- very changed.
+    KEY_LIFETIME = 3*30*24*60*60
+    MIN_KEY_LIFETIME_TO_USE = 30*24*60*60
+    
     def __init__(self, keyDir, passwordManager=None):
         """DOCDOC"""
         if passwordManager is None:
             passwordManager = mixminion.ClientUtils.CLIPasswordManager()
         createPrivateDir(keyDir)
-        fn = os.path.join(keyDir, "keyring")
-        self.keyring = mixminion.ClientUtils.LazyEncryptedStore(
-            fn, passwordManager, pwdName="ClientKeyring",
-            queryPrompt="Enter password for keyring:",
-            newPrompt="Entrer new keyring password:",
-            magic="KEYRING1",
-            initFn=lambda:{})
+        obsoleteFn = os.path.join(keyDir, "keyring")
+        if os.path.exists(obsoleteFn):
+            LOG.warn("Ignoring obsolete keyring stored in %r",obsoleteFn)
+        fn = os.path.join(keyDir, "keyring.txt")
+        self.keyring = mixminion.ClientUtils.Keyring(fn, passwordManager)
 
-    def _getKey(self, keyid, create=0, createFn=None, password=None):
+    def getSURBKey(self, name="", create=0, password=None):
         """Helper function. Return a key for a given keyid.
 
            keyid -- the name of the key.
            create -- If true, create a new key if none is found.
            createFn -- a callback to return a new key.
            password -- Optionally, a password for the keyring.
+           DOCDOC
         """
         if not self.keyring.isLoaded():
             try:
@@ -92,42 +95,33 @@
                 return None
             if not self.keyring.isLoaded():
                 return None
+
         try:
-            return self.keyring.get()[keyid]
-        except KeyError:
-            if not create:
+            key = self.keyring.getNewestSURBKey(
+                name,minLifetime=self.MIN_KEY_LIFETIME_TO_USE)
+            if key:
+                return key
+            elif not create:
                 return None
             else:
-                LOG.info("Creating new key for identity %r", keyid)
-                key = createFn()
-                self.keyring.get()[keyid] = key
+                # No key, we're allowed to create.
+                LOG.info("Creating new key for identity %r", name)
+                return self.keyring.newSURBKey(name,
+                                               time.time()+self.KEY_LIFETIME)
+        finally:
+            if self.keyring.isDirty():
                 self.keyring.save()
-                return key
-
-    def getSURBKey(self, name="", create=0, password=None):
-        """Return the key for a given SURB identity."""
-        k = self._getKey("SURB-"+name,
-                        create=create, 
-                         createFn=lambda: mixminion.Crypto.trng(20),
-                        password=password)
-        if k is not None and len(k) != 20:
-            raise MixError("Bad length on SURB key")
-        return k
-
-    def getSURBKeys(self, name="", password=None):
-        """Return the keys for _all_ SURB identities as a map from
-           name to key."""
+        
+    def getSURBKeys(self, password=None):
+        """Return the keys for _all_ SURB identities as a list of
+           (name,key) tuples."""
         try:
             self.keyring.load(create=0,password=password)
         except mixminion.ClientUtils.BadPassword:
             LOG.error("Incorrect password")
-        if not self.keyring.isLoaded(): return {}
-        r = {}
-        d = self.keyring.get()
-        for k,v in d.items():
-            if k.startswith("SURB-"):
-                r[k[5:]] = v
-        return r
+        if not self.keyring.isLoaded(): return []
+        if self.keyring.isDirty(): self.keyring.save()
+        return self.keyring.getAllSURBKeys()
 
 def installDefaultConfig(fname):
     """Create a default, 'fail-safe' configuration in a given file"""
@@ -306,7 +300,7 @@
             payloads = mixminion.BuildMessage.encodeMessage(message, 0,
                                 fragmentedMessagePrefix)
             if len(payloads) > 1:
-                address.setFragmented(not noSSFragmented, len(payloads))
+                address.setFragmented(not noSSFragments, len(payloads))
             else:
                 address.setFragmented(0,1)
         else:
@@ -597,6 +591,7 @@
     configFile = os.path.expanduser(configFile)
 
     if not os.path.exists(configFile):
+        print >>sys.stderr,"Writing default configuration file to %r"%configFile
         installDefaultConfig(configFile)
 
     try:
@@ -994,7 +989,8 @@
              ["help", "verbose", "config=", "download-directory=",
               "to=", "hops=", "path=", "reply-block=",
               "input=", "queue", "no-queue",
-              "subject=", "from=", "in-reply-to=", "references=", ])
+              "subject=", "from=", "in-reply-to=", "references=",
+              "deliver-fragments" ])
 
     if not options:
         sendUsageAndExit(cmd)
@@ -1013,7 +1009,7 @@
             h_irt = val
         elif opt == '--references':
             h_references = val
-        elif opt == '?????????':
+        elif opt == '--deliver-fragments':
             no_ss_fragment = 1
 
     if args:
@@ -1101,7 +1097,7 @@
         client.sendForwardMessage(
             parser.directory, parser.exitAddress, parser.pathSpec,
             message, parser.startAt, parser.endAt, forceQueue, forceNoQueue,
-            no_ss_fragment)
+            forceNoServerSideFragments=no_ss_fragment)
 
 _PING_USAGE = """\
 Usage: mixminion ping [options] serverName
@@ -1665,6 +1661,21 @@
 
     try:
         clientLock()
-        client.queue.inspectQueue()
+        res = client.queue.inspectQueue()
     finally:
         clientUnlock()
+
+    if not res:
+        print "(No packets in queue)"
+        return
+
+    res_items = [ (displayServer(ri),count,date)
+                  for ri,(count,date) in res.items() ]
+    res_items.sort()
+    now = time.time()
+    for server, count, date in res_items:
+        days = floorDiv(now-date, 24*60*60)
+        if days < 1:
+            days = "<1"
+        print "%2d packets for %s (oldest is %s days old)"%(
+            count, server, days)

Index: ClientUtils.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientUtils.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- ClientUtils.py	19 Nov 2003 09:48:09 -0000	1.9
+++ ClientUtils.py	20 Nov 2003 08:51:28 -0000	1.10
@@ -16,12 +16,15 @@
 import os
 import sys
 import time
+import struct
 
-import mixminion.Crypto
 import mixminion.Filestore
 
-from mixminion.Common import LOG, MixError, UIError, createPrivateDir, \
-     floorDiv, previousMidnight, readFile, writeFile
+from mixminion.Common import LOG, MixError, UIError, ceilDiv, \
+     createPrivateDir, floorDiv, previousMidnight, readFile, \
+     succeedingMidnight, writeFile, armorText, unarmorText
+from mixminion.Crypto import sha1, ctr_crypt, DIGEST_LEN, AES_KEY_LEN, \
+     getCommonPRNG, trng
 from mixminion.ServerInfo import displayServer
 
 #----------------------------------------------------------------------
@@ -123,7 +126,7 @@
 
 def getNewPassword_term(prompt):
     """Read a new password from the console, then return it."""
-    s2 = "Verify password:".rjust(len(prompt))
+    s2 = "Verify passphrase:".rjust(len(prompt))
     if os.isatty(sys.stdout.fileno()):
         f = sys.stdout
     else:
@@ -133,70 +136,96 @@
         p2 = getPassword_term(s2)
         if p1 == p2:
             return p1
-        f.write("Passwords do not match.\n")
+        f.write("Passphrases do not match.\n")
         f.flush()
 
 #----------------------------------------------------------------------
 # Functions to save and load data do disk in password-encrypted files.
 #
-# The file format is:
-#     variable      [File-specific magic, used to make sure we have an
-#                    encrypted file.]
-#     8 bytes       [Random salt.]
-#     variable      [AES-CTR Encrypted data:
-#                             key is sha1(salt+password+salt),
-#                             value is data+sha1(data+salt+magic)]
-#
-# Note that this format does not conceal the length of the data.
+# The file format is documented in E2E-spec.txt.
 
-def readEncryptedFile(fname, password, magic):
+MAGIC_LEN = 8
+SALT_LEN = 8
+
+def _readEncryptedFile(fname, password, magicList):
     """Read encrypted data from the file named 'fname', using the password
-       'password' and checking for the filetype 'magic'.  Returns the 
-       plaintext file contents on success.  If the 
+       'password' and checking for a magic string contained in 'magicList'.
+       Returns the magic string and the plaintext file contents on success.
 
        If the file is corrupt or the password is wrong, raises BadPassword.
        If the magic is incorrect, raises ValueError.
     """
-    s = readFile(fname, 1)
-    if not s.startswith(magic):
+    assert list(map(len, magicList)) == [8]*len(magicList)
+    
+    text = readFile(fname)
+    r = unarmorText(text, ["TYPE III KEYRING"])
+    if len(r) != 1:
+        raise ValueError("Bad ascii armor on keyring")
+    tp, headers, s = r[0]
+    assert tp == "TYPE III KEYRING"
+    vers = [ v for k,v in headers if k == 'Version' ]
+    if not vers or vers[0] != '0.1':
+        raise ValueError("Unrecognized version on keyring")
+    
+    if len(s) < MAGIC_LEN+1 or s[MAGIC_LEN] != '\x00':
+        raise ValueError("Unrecognized encryption format on %s"%fname)
+    if s[:MAGIC_LEN] not in magicList:
         raise ValueError("Invalid versioning on %s"%fname)
-    s = s[len(magic):]
+    magic = s[:8]
+    s = s[MAGIC_LEN+1:]
     if len(s) < 28:
-        raise MixError("File %s too short."%fname)
-    salt = s[:8]
-    s = s[8:]
-    key = mixminion.Crypto.sha1(salt+password+salt)[:16]
-    s = mixminion.Crypto.ctr_crypt(s, key)
-    data = s[:-20]
-    hash = s[-20:]
-    if hash != mixminion.Crypto.sha1(data+salt+magic):
+        raise MixError("File %s is too short."%fname)
+    salt = s[:SALT_LEN]
+    s = s[SALT_LEN:]
+    key = sha1(salt+password+salt)[:AES_KEY_LEN]
+    s = ctr_crypt(s, key)
+    data = s[:-DIGEST_LEN]
+    hash = s[-DIGEST_LEN:]
+    if hash != sha1(data+salt+magic):
         raise BadPassword()
-    return data
 
-def writeEncryptedFile(fname, password, magic, data):
+    # We've decrypted it; now let's extract the data from the padding.
+    if len(data) < 4:
+        raise MixError("File %s is too short"%fname)
+    length, = struct.unpack("!L", data[:4])
+    if len(data) < length+4:
+        raise MixError("File %s is too short"%fname)
+
+    return magic, data[4:4+length]
+
+def _writeEncryptedFile(fname, password, magic, data):
     """Write 'data' into an encrypted file named 'fname', replacing it
        if necessary.  Encrypts the data with the password 'password',
        and uses the filetype 'magic'."""
-    salt = mixminion.Crypto.getCommonPRNG().getBytes(8)
-    key = mixminion.Crypto.sha1(salt+password+salt)[:16]
-    hash = mixminion.Crypto.sha1("".join([data,salt,magic]))
-    encrypted = mixminion.Crypto.ctr_crypt(data+hash, key)
-    writeFile(fname, "".join([magic,salt,encrypted]), binary=1)
-
-def readEncryptedPickled(fname, password, magic):
-    """Read the pickled object stored in the encrypted file 'fname'. Arguments
-       are as for 'readEncryptedFile'."""
-    return cPickle.loads(readEncryptedFile(fname, password, magic))
-
-def writeEncryptedPickled(fname, password, magic, obj):
-    """Write 'obj' into encrypted file 'fname'. Arguments are as for
-       'writeEncryptedFile'."""
-    data = cPickle.dumps(obj, 1)
-    writeEncryptedFile(fname, password, magic, data)
+    assert len(magic) == MAGIC_LEN
+    prng = getCommonPRNG()
+    length = struct.pack("!L", len(data))
+    paddingLen = ceilDiv(len(data), 1024)*1024 - len(data)
+    padding = prng.getBytes(paddingLen)
+    data = "".join([length,data,padding])
+    salt = prng.getBytes(SALT_LEN)
+    key = sha1(salt+password+salt)[:AES_KEY_LEN]
+    hash = sha1("".join([data,salt,magic]))
+    encrypted = ctr_crypt(data+hash, key)
+    contents = "".join([magic,"\x00",salt,encrypted])
+    writeFile(fname, armorText(contents,
+                               "TYPE III KEYRING", [("Version","0.1")]))
 
-class LazyEncryptedStore:
+class _LazyEncryptedStore:
     """Wrapper for a file containing an encrypted object, to
        perform password querying and loading on demand."""
+    ## Fields:
+    # fname, pwdManager, pwdName, queryPrompt, newPrompt, initFn:
+    #    As documented in __init__.
+    # okMagic: A list of magic strings we're willing to accept on files
+    #    we're reading.
+    # bestMagic: The magic string we use on files we're writing.
+    # obsoleteMagic: A list of magic strings which we flag as "obsolete"
+    #    instead of "unrecongized" when giving error messages to the user.
+    # password: The cached password for this object
+    # object: The cached contents of this object, or None if this object
+    #    hasn't been loaded.
+    # loaded: Flag: has this object been loaded?
     def __init__(self, fname, pwdManager, pwdName, queryPrompt, newPrompt,
                  magic, initFn):
         """Create a new LazyEncryptedStore
@@ -213,24 +242,36 @@
         self.pwdName = pwdName
         self.queryPrompt = queryPrompt
         self.newPrompt = newPrompt
-        self.magic = magic
         self.object = None
         self.loaded = 0
         self.password = None
+        self.okMagic = [magic]
+        self.bestMagic = magic
+        assert len(magic) == MAGIC_LEN
         self.initFn = initFn
+        self.obsoleteMagic = [] 
+        
     def load(self, create=0,password=None):
         """Try to load the encrypted file from disk.  If 'password' is
            not provided, query it from the password manager.  If the file
            does not exist, and 'create' is true, get a new password and
            create the file."""
         if self.loaded:
+            # No need to re-load an already-loaded object.
             return 
         elif os.path.exists(self.fname):
-            if not readFile(self.fname).startswith(self.magic):
-                raise MixError("Unrecognized versioning on file %s"%self.fname)
+            # Okay, the file is there. Snarf it from disk and try to give a
+            # good warning for its magic string.
+            contents = readFile(self.fname)
+##             if contents[:8] in self.obsoleteMagic:
+##                 raise MixError("Found an obsolete keyring at %r.  Remove this file to use SURBs with this version of Mixminion."%self.fname)
+##             if len(contents)<8 or contents[:8] not in self.okMagic:
+##                 raise MixError("Unrecognized versioning on file %s"%self.fname)
+            
             # ... see if we can load it with no password ...
             if self._loadWithPassword(""):
                 return
+            # Nope; see if we can use a password we were given.
             if password is not None:
                 self._loadWithPassword(password)
                 if not self.loaded:
@@ -240,6 +281,7 @@
                 self.pwdManager.getPassword(self.pwdName, self.queryPrompt,
                                             self._loadWithPassword)
         elif create:
+            # It isn't there, but we're allowed to create it.
             if password is not None:
                 self.password = password
             else:
@@ -250,42 +292,223 @@
             self.save()
         else:
             return
+
     def _loadWithPassword(self, password):
         """Helper function: tries to load the file with a given password.
            If Successful, return 1. Else return 0."""
         try:
-            self.object = readEncryptedPickled(self.fname,password,self.magic)
+            m, val = _readEncryptedFile(self.fname,password, self.okMagic+self.obsoleteMagic)
+            if m in self.obsoleteMagic:
+                raise MixError("Found an obsolete keyring at %r.  Remove this file to use SURBs with this version of Mixminion."%self.fname)
+            self._decode(val, m)
             self.password = password
             self.loaded = 1
             return 1
         except MixError:
             return 0
+
     def isLoaded(self):
         """Return true iff this file has been successfully loaded."""
         return self.loaded
+
     def get(self):
         """Returns the contents of this file. The file must first have
            been loaded."""
         assert self.loaded
         return self.object
+
     def set(self, val):
         """Set the contents of this file.  Does not save the file to
            disk."""
         self.object = val
         self.loaded = 1
+
     def setPassword(self, pwd):
         """Set the password on this file."""
         self.password = pwd
         self.pwdManager.setPassword(self.pwdName, pwd)
+
     def save(self):
         """Flush the current contens of this file to disk."""
         assert self.loaded and self.password is not None
-        writeEncryptedPickled(self.fname, self.password, self.magic,
-                              self.object)
-    def _encode(self,obj):
-        return cPickle.dumps(obj,1)
-    def _decode(self,obj):
-        return cPickle.loads(obj,1)
+        _writeEncryptedFile(self.fname, self.password, self.bestMagic,
+                            self._encode())
+
+    def _encode(self):
+        """Helper function for subclasses to override: convert self.object to a
+           string for storage, and return the converted object."""
+        return cPickle.dumps(self.object, 1)
+    
+    def _decode(self,val,magic):
+        """Helper function: given a decrypted string and magic string, sets
+           self.object to the corresponding decoded value."""
+        self.object = cPickle.loads(val)
+
+class _KeyringImpl:
+    """Helper class: serves as the value stored by Keyring.  Contains a bunch
+       of SURB keys and unrecognized key data, along with functions to
+       manipulate those SURB keys.
+
+       Uses the file format documented in appendix A.2 of E2E-spec.txt
+    """
+    ## Fields
+    # recognized: A list of (tp, val) tuples for every item in the keyring
+    #    whose type we recognize.
+    # unrecognized: A list of (tp, val) tuples for every item in the keyring
+    #    whose type we don't recognize.
+    # dirty: Boolean: does the state of this object match what we loaded
+    #    from disk?
+    # surbKeys: A map from lowercase keyid to a list of (expiry-time, secret)
+    #    for all of the SURB keys in the keyring.
+    SURB_KEY_TYPE = 0x00
+    def __init__(self, input="", now=None):
+        """Initialize this keyring representation from the encoded string
+           'input'.  If any keys are set to expire before 'now', delete them.
+        """
+        if now is None: now = time.time()
+
+        # Build lists of recongized and unrecognized items in 'input'.
+        self.unrecognized = []
+        rec = []
+        self.dirty = 0
+        while input:
+            if len(input) < 3:
+                raise MixError("Corrupt keyring: truncated entry.")
+            tp,length = struct.unpack("!BH", input[:3])
+            if len(input) < 3+length:
+                raise MixError("Corrupt keyring: truncated entry.")
+            val = input[3:3+length]
+            if tp == self.SURB_KEY_TYPE:
+                rec.append((tp,val))
+            else:
+                self.unrecognized.append((tp,val))
+            input = input[3+length:]
+
+        # Now, extract all the SURB keys from the keyring, and remove all
+        # expired SURB keys from self.recognized.
+        self.surbKeys = {}
+        self.recognized = []
+        for tp,val in rec:
+            if len(val) < 5 or '\0' not in val[4:]:
+                raise MixError("Truncated SURB key")
+            expiry, = struct.unpack("!L", val[:4])
+            if expiry < now:
+                self.dirty = 1
+            else:
+                self.recognized.append((tp,val))
+                val = val[4:]
+                identity = val[:val.index('\0')].lower()
+                secret = val[val.index('\0')+1:]
+                self.surbKeys.setdefault(identity,[]).append((expiry,secret))
+
+    def pack(self):
+        """Return a string representation of this keyring."""
+        items = self.recognized+self.unrecognized
+        # Scramble all the items, just to make sure that no broken
+        # implementations rely on their oreder.
+        getCommonPRNG().shuffle(items)
+        encoded = []
+        for tp, val in items:
+            encoded.append(struct.pack("!BH", tp, len(val)))
+            encoded.append(val)
+        return "".join(encoded)
+
+    def newSURBKey(self, identity, expiresAt, secretLen):
+        """See ClientUtils.Keyring.newSURBKey"""
+        assert '\0' not in identity
+        identity = identity.lower()
+        expires = succeedingMidnight(expiresAt)
+        secret = trng(secretLen)
+        encoded = "%s%s\0%s" % (struct.pack("!L", expires),identity,secret)
+        self.recognized.append((self.SURB_KEY_TYPE, encoded))
+        self.surbKeys.setdefault(identity, []).append((expires,secret))
+        self.dirty = 1
+        return secret
+
+    def getNewestSURBKey(self, identity, minLifetime, now=None):
+        """See ClientUtils.Keyring.getNewestSURBKey"""
+        identity = identity.lower()
+        if now is None:
+            now = time.time()
+        v = self.surbKeys.get(identity,[])
+        if not v:
+            return None
+        v.sort()
+        expires, secret = v[-1]
+        if expires < now+minLifetime:
+            return None
+        return secret
+
+    def getAllSURBKeys(self):
+        """See ClientUtils.Keyring.getAllSURBKeys"""
+        res = []
+        for identity, lst in self.surbKeys.items():
+            for _, secret in lst:
+                res.append((identity, secret))
+        return res
+
+class Keyring(_LazyEncryptedStore):
+    """Class to wrap a lazy-loaded file holding a bundle of SURB keys for
+       a client.  The format is as described in E2E-spec.txt, appendix A.2."""
+    def __init__(self, fname, pwdManager):
+        """Create a new LazyEncryptedStore
+              fname -- The name of the file to hold the encrypted object.
+              pwdManager -- A PasswordManager instance.
+        """
+        _LazyEncryptedStore.__init__(self,
+            fname, pwdManager, pwdName="ClientKeyring",
+            queryPrompt = "Enter passphrase for keyring:",
+            newPrompt = "Enter new passphrase for client keyring:",
+            magic = "KEYRING2",
+            initFn = _KeyringImpl)
+        self.obsoleteMagic = [ "KEYRING1" ]
+    def _encode(self):
+        return self.object.pack()
+    def _decode(self,val,magic):
+        assert magic == 'KEYRING2'
+        self.object = _KeyringImpl(val,now=self._now)
+    def newSURBKey(self, identity, expiresAt, secretLen=DIGEST_LEN):
+        """Generate a fresh SURB key for the identity 'identity',
+           set to expire on the time 'expiresAt', and returns the freshly
+           generated key.  Old keys are not replaced, and the new key
+           will not be saved until you call save() on this object.
+        """
+        return self.object.newSURBKey(identity,expiresAt,secretLen)
+    def getNewestSURBKey(self, identity, minLifetime=2*24*60*60, now=None):
+        """Return the SURB key for the identity 'identity' that has
+           the latest expiration date.  If no such key exists, or that
+           key would expire in less than 'minLifetime' seconds after
+           'now', return None.
+        """
+        return self.object.getNewestSURBKey(identity, minLifetime, now)
+    def getAllSURBKeys(self):
+        """Return a list of (identity,key) tuples for every SURK key in
+           this keyring.
+        """
+        return self.object.getAllSURBKeys()
+    def isDirty(self):
+        """Return true iff this keyring contains state the has not been
+           written to disk.
+        """
+        return self.object.dirty
+    def save(self):
+        _LazyEncryptedStore.save(self)
+        self.object.dirty = 0
+    def load(self, create=0, password=None, now=None):
+        """Try to load the encrypted keyring from disk.  If 'password' is
+           not provided, query it from the password manager.  If the file
+           does not exist, and 'create' is true, get a new password and
+           create the file.
+
+           If the keyring contains any expired keys, remove them.  (They
+           will not be removed from disk until this keyring is next
+           save()d.)
+        """
+        self._now = now
+        try:
+            _LazyEncryptedStore.load(self, create=create, password=password)
+        finally:
+            del self._now
         
 # ----------------------------------------------------------------------
 
@@ -314,9 +537,12 @@
         self.sync()
 
     def findUnusedSURBs(self, surbList, nSURBs=1, verbose=0, now=None):
-        """Given a list of ReplyBlock objects, find the first that is neither
-           expired, about to expire, or used in the past.  Return None if
-           no such reply block exists. DOCDOC returns list, nSurbs"""
+        """Given a list of ReplyBlock objects, return a list of the first
+           'nSURBs' of them that neither are expired, are about to expire,
+           or have been used in the past.  If less than 'nSURBs' exist,
+           return as many as possible. If 'verbose' is true, log the status
+           of the SURBs considered.
+        """
         if now is None:
             now = time.time()
         nUsed = nExpired = nShortlived = 0
@@ -376,7 +602,7 @@
         self.sync()
 
     def _encodeKey(self, surb):
-        return binascii.b2a_hex(mixminion.Crypto.sha1(surb.pack()))
+        return binascii.b2a_hex(sha1(surb.pack()))
     def _encodeVal(self, timestamp):
         return str(timestamp)
     def _decodeVal(self, timestamp):
@@ -482,15 +708,15 @@
         """Remove the packet named with the handle 'handle'."""
         self.store.removeMessage(handle)
 
-    def inspectQueue(self, now=None):
-        """Print a message describing how many packets in the queue are headed
-           to which addresses."""
-        #XXXX006 refactor
-        if now is None:
-            now = time.time()
+    def inspectQueue(self):
+        """Return a dict from routinginfo to a tuple of: (n,t), where
+           n is the number of packets waiting for that routinginfo, and
+           t is the insertion-data of the oldest packet waiting for that
+           routinginfo.
+        """
         handles = self.getHandles()
         if not handles:
-            print "[Queue is empty.]"
+            return {}
             return
         self.loadMetadata()
         timesByServer = {}
@@ -500,14 +726,12 @@
             except mixminion.Filestore.CorruptedFile:
                 continue
             timesByServer.setdefault(routing, []).append(when)
+        res = {}
         for s in timesByServer.keys():
             count = len(timesByServer[s])
             oldest = min(timesByServer[s])
-            days = floorDiv(now - oldest, 24*60*60)
-            if days < 1:
-                days = "<1"
-            print "%2d packets for %s (oldest is %s days old)"%(
-                count, displayServer(s), days)
+            res[s] = (count, oldest)
+        return res
 
     def cleanQueue(self, maxAge=None, now=None):
         """Remove all packets older than maxAge seconds from this queue."""

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.165
retrieving revision 1.166
diff -u -d -r1.165 -r1.166
--- test.py	19 Nov 2003 09:48:10 -0000	1.165
+++ test.py	20 Nov 2003 08:51:28 -0000	1.166
@@ -20,6 +20,7 @@
 import re
 import socket
 import stat
+import struct
 import sys
 import threading
 import time
@@ -1200,14 +1201,6 @@
             tot += v
         self.failUnless(4900<tot<5100)
 
-##      itot=ftot=0
-##      for i in xrange(1000000):
-##          itot += PRNG.getInt(10)
-##          ftot += PRNG.getFloat()
-
-##      print "AVG INT", itot/1000000.0
-##      print "AVG FLT", ftot/1000000.0
-
         for i in xrange(100):
             self.failUnless(0 <= PRNG.getFloat() < 1)
 
@@ -2416,16 +2409,6 @@
         self.assertEquals(payload,
              BuildMessage._decodeEncryptedForwardPayload(efwd_p,efwd_t,rsa1).getUncompressedContents())
 
-##      # Stateful reply
-##      secrets = [ "Is that you, Des","troyer?Rinehart?" ]
-##      sdict = { 'tag1'*5 : secrets }
-##      ks = Keyset(secrets[1])
-##      m = lioness_decrypt(encoded1, ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE))
-##      ks = Keyset(secrets[0])
-##      m = lioness_decrypt(m, ks.getLionessKeys(PAYLOAD_ENCRYPT_MODE))
-##      self.assertEquals(payload, BuildMessage._decodeReplyPayload(m,secrets))
-##      repl1 = m
-
         # Stateless reply
         tag = "To light my way out\xBE"
         passwd = "out I would have to burn every paper in the briefcase"
@@ -2449,19 +2432,16 @@
         decodePayload = BuildMessage.decodePayload
         # fwd
         for pk in (self.pk1, None):
-            ##for d in (sdict, None): # stateful replies disabled.
-                for p in (passwd, None):
-                    for tag in ("zzzz"*5, "pzzz"*5):
-                        self.assertPayloadDecodesTo(payload,
-                                                    encoded1, tag, pk, p)
+            for p in (passwd, None):
+                for tag in ("zzzz"*5, "pzzz"*5):
+                    self.assertPayloadDecodesTo(payload,
+                                                encoded1, tag, pk, p)
 
         # efwd
-        ##for d in (sdict, None): # stateful replies disabled
-        if 1:
-            for p in (passwd, None):
-                self.assertPayloadDecodesTo(payload, efwd_p, efwd_t, rsa1, p)
-                self.assertPayloadDecodesTo(None, efwd_p, efwd_t, None, p)
-                self.assertPayloadDecodesTo(None, efwd_p, efwd_t, self.pk2, p)
+        for p in (passwd, None):
+            self.assertPayloadDecodesTo(payload, efwd_p, efwd_t, rsa1, p)
+            self.assertPayloadDecodesTo(None, efwd_p, efwd_t, None, p)
+            self.assertPayloadDecodesTo(None, efwd_p, efwd_t, self.pk2, p)
 
         # Stateful replies are disabled.
 
@@ -4235,7 +4215,7 @@
         tm = C._parseTime("2001-12-25 06:15:10.623")
         self.assertEquals(time.gmtime(tm)[:6], (2001,12,25,6,15,10))
         # nicknames
-        self.assertEquals(C._parseNickname("Mrs.Premise"), "Mrs.Premise")
+        self.assertEquals(C._parseNickname("Mrs-Premise"), "Mrs-Premise")
         # Filenames
         self.assertEquals(C._parseFilename(" ab/c/d"), "ab/c/d")
         self.assertEquals(C._parseFilename("  ~/ab/c/d"),
@@ -4307,6 +4287,7 @@
         fails(C._parseDate, "2000-50-01 12:12:12.3")
         fails(C._parseTime, "2000/50/01 12:12:99")
         fails(C._parseNickname, "Mrs Premise")
+        fails(C._parseNickname, "-Mrs-Premise")
         fails(C._parseNickname, "../../../AllYourBase")
         fails(C._parseNickname, "Z"*129)
         fails(C._parseNickname, ""*129)
@@ -4371,7 +4352,7 @@
 EncryptPrivateKey: no
 Homedir: %s
 Mode: relay
-Nickname: The_Server
+Nickname: The-Server
 Contact-Email: a@b.c
 Comments: This is a test of the emergency
    broadcast system
@@ -4439,7 +4420,7 @@
         eq(info['Server']['Descriptor-Version'], "0.2")
         eq(info['Incoming/MMTP']['IP'], "192.168.0.1")
         eq(info['Incoming/MMTP']['Hostname'], "Theserver")
-        eq(info['Server']['Nickname'], "The_Server")
+        eq(info['Server']['Nickname'], "The-Server")
         self.failUnless(0 <= time.time()-info['Server']['Published'] <= 120)
         self.failUnless(0 <= time.time()-info['Server']['Valid-After']
                           <= 24*60*60)
@@ -6167,38 +6148,44 @@
         d = mix_mktemp()
         createPrivateDir(d)
         f1 = os.path.join(d, "foo")
+        magic1 = "ABCDEFGH"
         # Test reading and writing.
-        CU.writeEncryptedFile(f1, password="x", magic="ABC", data="xyzzyxyzzy")
+        CU._writeEncryptedFile(f1,password="x",magic=magic1,data="xyzzyxyzzy")
         contents = readFile(f1)
-        self.assertEquals(contents[:3], "ABC")
-        salt = contents[3:11]
+        self.assert_(contents.startswith(
+            "-----BEGIN TYPE III KEYRING-----\n"
+            "Version: 0.1\n\n"))
+        self.assert_(contents.endswith(
+            "\n-----END TYPE III KEYRING-----\n"))
+        idx1 = contents.index("\n\n")+2
+        idx2 = contents.rindex("\n-----END TYPE")
+        contents = base64.decodestring(contents[idx1:idx2])
+        
+        self.assertEquals(contents[:8], magic1)
+        self.assertEquals(contents[8], '\x00')
+        salt = contents[9:17]
         key = mixminion.Crypto.sha1(salt+"x"+salt)[:16]
-        decrypted = mixminion.Crypto.ctr_crypt(contents[11:], key)
-        self.assertEquals(decrypted, "xyzzyxyzzy"+mixminion.Crypto.sha1(
-            "xyzzyxyzzy"+salt+"ABC"))
+        decrypted = mixminion.Crypto.ctr_crypt(contents[17:], key)
+        self.assertEquals(len(decrypted), 4+1024+20)
+        self.assertEquals(decrypted[:4], "\x00\x00\x00\x0A")
+        self.assertEquals(decrypted[4:14], "xyzzyxyzzy")
+        self.assertEquals(decrypted[-20:],
+                          mixminion.Crypto.sha1(decrypted[:-20]+salt+magic1))
+        
+        self.assertEquals((magic1,"xyzzyxyzzy"),
+                          CU._readEncryptedFile(f1, "x", [magic1, "BLIZNERT"]))
         
-        self.assertEquals("xyzzyxyzzy",
-              CU.readEncryptedFile(f1, "x", "ABC"))
-
         # Try reading with wrong password.
-        self.assertRaises(CU.BadPassword, CU.readEncryptedFile,
-                          f1, "nobodaddy", "ABC")
+        self.assertRaises(CU.BadPassword, CU._readEncryptedFile,
+                          f1, "nobodaddy", [magic1])
 
         # Try reading with wrong magic.
-        self.assertRaises(ValueError, CU.readEncryptedFile,
-                          f1, "x", "ABX")
+        self.assertRaises(ValueError, CU._readEncryptedFile,
+                          f1, "x", ["XXXXYYYY"])
 
         # Try empty data.
-        CU.writeEncryptedFile(f1, password="x", magic="ABC", data="")
-        self.assertEquals("", CU.readEncryptedFile(f1, "x", "ABC"))
-        
-        # Test pickles.
-        f2 = os.path.join(d, "bar")
-        CU.writeEncryptedPickled(f2, "pswd", "ZZZ", [1,2,3])
-        self.assertEquals([1,2,3],CU.readEncryptedPickled(f2,"pswd","ZZZ"))
-        CU.writeEncryptedPickled(f2, "pswd", "ZZZ", {9:10,11:12})
-        self.assertEquals({9:10,11:12},
-                          CU.readEncryptedPickled(f2,"pswd","ZZZ"))
+        CU._writeEncryptedFile(f1, password="x", magic=magic1, data="")
+        self.assertEquals((magic1,""),CU._readEncryptedFile(f1, "x", [magic1]))
         
         # Test LazyEncryptedStore
         class DummyPasswordManager(CU.PasswordManager):
@@ -6211,9 +6198,10 @@
                 return self.d.get(name)
 
         f3 = os.path.join(d, "Baz")
+        magic0 = "MAGIC000"
         dpm = DummyPasswordManager({"Password1" : "p1"})
-        lep = CU.LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
-                                    "magic0", lambda: "x"*3)
+        lep = CU._LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
+                                     magic0, lambda: "x"*3)
         # Don't create.
         self.assert_(not lep.isLoaded())
         lep.load(create=0)
@@ -6221,15 +6209,87 @@
         lep.load(create=1)
         self.assert_(lep.isLoaded())
         self.assertEquals("x"*3, lep.get())
-        self.assertEquals("x"*3, CU.readEncryptedPickled(f3,"p1","magic0"))
+        m,v = CU._readEncryptedFile(f3,"p1",[magic0])
+        self.assertEquals((magic0,"x"*3), (m,cPickle.loads(v)))
 
-        lep = CU.LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
-                                     "magic0", lambda: "x"*3)
+        lep = CU._LazyEncryptedStore(f3, dpm, "Password1", "Q:", "N:",
+                                     magic0, lambda: "x"*3)
         lep.load()
         self.assertEquals("x"*3, lep.get())
         dpm.d = {}
         self.assertEquals("x"*3, lep.get())
 
+        # Finally, test keyring.
+        f4 = os.path.join(d, "keyring")
+        dpm.d = {"ClientKeyring" : "ckr" }
+        kr = CU.Keyring(f4, dpm)
+        kr.load(create=1) # We should already have the password cached.
+        # Test empty keyring.
+        self.assertEquals(kr.getAllSURBKeys(), [])
+        self.assertEquals(kr.getNewestSURBKey(""), None)
+        self.assert_(not kr.isDirty())
+        kr.save()
+        self.assertEquals(("KEYRING2", ""),
+                          CU._readEncryptedFile(f4, "ckr", ["KEYRING2"]))
+        kr = CU.Keyring(f4, dpm)
+        kr.load()
+        self.assertEquals(kr.getAllSURBKeys(), [])
+
+        # Now generate a few keys.
+        now = time.time()
+        month1 = now+30*24*60*60
+        month2 = now+60*24*60*60
+        month1d = succeedingMidnight(month1)
+        month2d = succeedingMidnight(month2)
+        alice1 = kr.newSURBKey("Alice", month1)
+        alice2 = kr.newSURBKey("Alice", month2)
+        bob = kr.newSURBKey("Bobby",month1)
+        self.assertEquals(len(alice1), 20)
+        self.assertEquals(len(bob), 20)
+        self.assertNotEquals(alice1, alice2)
+        self.assert_(kr.isDirty())
+        for i in (1,2):
+            self.assertEquals(alice2, kr.getNewestSURBKey("alice"))
+            self.assertUnorderedEq([("alice",alice1), ("alice",alice2),
+                                    ("bobby",bob)], kr.getAllSURBKeys())
+            self.assertEquals(None, kr.getNewestSURBKey("alice",
+                                               minLifetime=70*24*60*60))
+            if i == 1:
+                kr.save()
+                kr = CU.Keyring(f4, dpm)
+                kr.load()
+
+        # Check that contents are correctly encoded.
+        _, contents = CU._readEncryptedFile(f4, "ckr", ["KEYRING2"])
+        ln = 33
+        self.assertEquals(len(contents), ln*3)
+        k1, k2, k3 = contents[0:ln], contents[ln:ln*2], contents[ln*2:]
+        ex1 = ("\x00" +  # It's a SURB key
+               "\x00\x1E" +  # The rest is 30 bytes long
+               struct.pack("!L",month1d) + "alice\x00" + alice1)
+        ex2 = ("\x00" +  # It's a SURB key
+               "\x00\x1E" +  # The rest is 30 bytes long
+               struct.pack("!L",month2d) + "alice\x00" + alice2)
+        ex3 = ("\x00" +  # It's a SURB key
+               "\x00\x1E" +  # The rest is 30 bytes long
+               struct.pack("!L",month1d) + "bobby\x00" + bob)
+        self.assertUnorderedEq([ex1,ex2,ex3],[k1,k2,k3])
+
+        # Now get fancier: Add an unrecognized type, but expire Alice1 and Bob.
+        dummy = "\xFF\x00\x1E"+("X"*30)
+        CU._writeEncryptedFile(f4, "ckr", "KEYRING2", contents+dummy)
+        kr = CU.Keyring(f4, dpm)
+        kr.load(now=now+33*24*60*60)
+        self.assert_(kr.isDirty())
+        self.assertEquals(kr.getAllSURBKeys(), [('alice', alice2)])
+        self.assertEquals(kr.object.unrecognized, [(0xFF, "X"*30)])
+        kr.save()
+        self.assert_(not kr.isDirty())
+        _, contents = CU._readEncryptedFile(f4, "ckr", ["KEYRING2"])
+        self.assertEquals(len(contents), 66)
+        k1,k2 = contents[0:ln],contents[ln:]
+        self.assertUnorderedEq([dummy,ex2], [k1,k2])
+
     def testSURBLog(self):
         brb = BuildMessage.buildReplyBlock
         SURBLog = mixminion.ClientUtils.SURBLog
@@ -6734,9 +6794,6 @@
         raises(MixError, ppath, ks, None, "Alice:Bob,Fred", mboxWithoutServer)
         # Two stars.
         raises(MixError, ppath, ks, None, "Alice,*,Bob,*,Joe", email)
-        # NHops mismatch -- no longer an error.
-        ##raises(MixError, ppath, ks, None, "Alice:Bob,Joe", email, nHops=2)
-        ##raises(MixError, ppath, ks, None, "Alice:Bob,Joe", email, nHops=4)
         # Nonexistent file
         raises(MixError, ppath, ks, None, "./Pierre:Alice,*", email)
 
@@ -6923,7 +6980,7 @@
         keydir = mix_mktemp()
         keyring = mixminion.ClientMain.ClientKeyring(keydir)
         # Check for some nonexistent keys.
-        self.assertEquals({}, keyring.getSURBKeys(password="pwd"))
+        self.assertEquals([], keyring.getSURBKeys(password="pwd"))
         self.assertEquals(None, keyring.getSURBKey(create=0))
         # Reload, try again:
         kfirst = None
@@ -6939,7 +6996,7 @@
             k2 = keyring.getSURBKey(name="Bob",create=1)
             self.assertEquals(20, len(k2))
             self.assertNotEquals(k1,k2)
-            self.assertEquals({"":k1,"Bob":k2}, keyring.getSURBKeys())
+            self.assertUnorderedEq([("",k1),("bob",k2)], keyring.getSURBKeys())
 
         # Incorrect password
         keyring = mixminion.ClientMain.ClientKeyring(keydir)
@@ -7307,7 +7364,7 @@
     tc = loader.loadTestsFromTestCase
 
     if 0:
-        suite.addTest(tc(ServerInfoTests))
+        suite.addTest(tc(ClientUtilTests))
         return suite
     testClasses = [MiscTests,
                    MinionlibCryptoTests,
@@ -7326,6 +7383,7 @@
                    EventStatsTests,
                    NetUtilTests,
                    DNSFarmTests,
+                   ClientUtilTests,
            
                    ModuleTests,
                    ClientDirectoryTests,