[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[minion-cvs] Directories implemented for client and server, but not ...



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv11194/minion/lib/mixminion

Modified Files:
	ClientMain.py Config.py Crypto.py Main.py ServerInfo.py 
	test.py 
Log Message:
Directories implemented for client and server, but not doc'd or tested.

TODO:
- Reflect state of directory work.

setup.py:
- Run python as python -O from the default-installed script.

ClientMain.py:
- Reimplement keystore to know about directories and select paths.  The new
  one also caches parsed values to run a bit faster.
- Make the path interface a little more complicated.
- Simplify 'main' a bit by refactoring out the configfile and usage logic.

Config, ServerInfo:
- Builtin gzip support

Config, ServerInfo, server/ServerConfig:
- Restrict nicknames to reasonable characters

Crypto:
- Add 'fingerprint' function
- Document more.
- Add rng.pick(lst) as a shortcut for lst[rng.getInt(len(lst))]

Main, directory/DirMain:
- Add CLI for directory generation:

ServerInfo:
- Add numerous helper functions to ServerInfo
- Add ServerDirectory class to parse server directories.

test:
- Add tests for nickname validation
- Tests for pk_same_public_key and pk_fingerprint
- Tests for directory generation and parsing.

directory/ServerList:
- Debug, document, refactor

server/Modules:
- Use balanced 'banners' around email-quoted messages (Suggested by Lucky)

server/ServerKeys:
- Add 'now' argument to generateServerDescriptorAndKeys for testing


Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- ClientMain.py	16 Dec 2002 02:40:11 -0000	1.17
+++ ClientMain.py	3 Jan 2003 05:14:47 -0000	1.18
@@ -6,8 +6,7 @@
    Code for Mixminion command-line client.
 
    NOTE: THIS IS NOT THE FINAL VERSION OF THE CODE.  It needs to
-         support replies and end-to-end encryption.  It also needs to
-         support directories.
+         support replies and end-to-end encryption.
    """
 
 __all__ = []
@@ -27,108 +26,312 @@
 #      - Per-system directory location is a neat idea, but individual users
 #        must check signature.  That's a way better idea for later.
 
+import cPickle
 import getopt
 import os
+import stat
 import sys
 import time
 import types
+import urllib
 
 import mixminion.BuildMessage
 import mixminion.Crypto
 import mixminion.MMTPClient
-from mixminion.Common import LOG, floorDiv, MixError, MixFatalError, \
-     createPrivateDir, isSMTPMailbox, formatDate
+from mixminion.Common import IntervalSet, LOG, floorDiv, MixError, \
+     MixFatalError, createPrivateDir, isSMTPMailbox, formatDate, \
+     formatFnameTime
 from mixminion.Config import ClientConfig, ConfigError
-from mixminion.ServerInfo import ServerInfo
+from mixminion.ServerInfo import ServerInfo, ServerDirectory
 from mixminion.Packet import ParseError, parseMBOXInfo, parseSMTPInfo, \
      MBOX_TYPE, SMTP_TYPE, DROP_TYPE
 
-class TrivialKeystore:
-    """This is a temporary keystore implementation until we get a working
-       directory server implementation.
+# FFFF This should be made configurable.
+MIXMINION_DIRECTORY_URL = "http://www.mixminion.net/directory/latest.gz";
+# FFFF This should be made configurable.
+MIXMINION_DIRECTORY_FINGERPRINT = ""
+                                
+class ClientKeystore:
+    """DOCDOC"""
+    #DOCDOC
+    ##Fields:
+    # dir
+    # lastModified
+    # lastDownload
+    # serverList: list of (ServerInfo, 'D'|'I:filename')
+    # byNickname:
+    # byCapability:
+    # allServers:
+    #     All are maps/lists of (ServerInfo, where)
+    # __scanning
+    ## Layout:
+    # DIR/cache
+    # DIR/dir.gz *or* DIR/dir
+    # DIR/servers/X
+    #             Y
+    def __init__(self, directory):
+        "DOCDOC"
+        self.dir = directory
+        createPrivateDir(self.dir)
+        self.__scanning = 0
+        self._load()
+    def downloadDirectory(self):
+        "DOCDOC"
+        #DOCDOC
+        opener = URLopener()
+        url = MIXMINION_DIRECTORY_URL
+        LOG.info("Downloading directory from %s", url)
+        infile = FancyURLopener().open(url)
+        if url.endswith(".gz"):
+            fname = os.path.join(self.dir, "dir_new.gz")
+            outfile = open(fname, 'wb')
+            gz = 1
+        else:
+            fname = os.path.join(self.dir, "dir_new")
+            outfile = open(fname, 'w')
+            gz = 0
+        while 1:
+            s = infile.read(1<<16)
+            if not s: break
+            outfile.write(s)
+        infile.close()
+        outfile.close()
+        LOG.info("Validating directory")
+        try:
+            directory = ServerDirectory(fname=fname)
+        except ConfigError, e:
+            raise MixFatalError("Downloaded invalid directory: %s" % e)
 
-       The idea couldn't be simpler: we just keep a directory of files, each
-       containing a single server descriptor.  We cache nothing; we validate
-       everything; we have no automatic path generation.  Servers can be
-       accessed by nickname, by filename within our directory, or by filename
-       from elsewhere on the filesystem.
+        identity = directory['Signature']['DirectoryIdentity']
+        fp = MIXMINION_DIRECTORY_FINGERPRINT
+        if fp and pk_fingerprint(identity) != fp:
+            raise MixFatalError("Bad identity key on directory")
 
-       We skip all server descriptors that have expired, that will
-       soon expire, or which aren't yet in effect.
-       """
-    ## Fields:
-    # directory: path to the directory we scan for server descriptors.
-    # byNickname: a map from nickname to valid ServerInfo object.
-    # byFilename: a map from filename within self.directory to valid
-    #     ServerInfo object.
-    def __init__(self, directory, now=None):
-        """Create a new TrivialKeystore to access the descriptors stored in
-           directory.  Selects descriptors that are valid at the time 'now',
-           or at the current time if 'now' is None."""
-        self.directory = directory
-        createPrivateDir(directory)
-        self.byNickname = {}
-        self.byFilename = {}
+        try:
+            os.unlink(os.path.join(self.dir, "cache"))
+        except OSError:
+            pass
 
-        if now is None:
-            now = time.time()
+        if gz:
+            os.rename(fname, os.path.join(self.dir, "dir.gz"))
+        else:
+            os.rename(fname, os.path.join(self.dir, "dir"))
 
-        for f in os.listdir(self.directory):
+        self.rescan()
+    def rescan(self, now=None):
+        "DOCDOC"
+        #DOCDOC
+        self.lastModified = self.lastDownload = -1
+        self.serverList = []
+        gzipFile = os.path.join(self.dir, "dir.gz")
+        dirFile = os.path.join(self.dir, "dir")
+        f = None
+        for fname in gzipFile, dirFile:
+            if not os.path.exists(fname): continue
+            self.lastDownload = self.lastModified = \
+                                os.stat(fname)[stat.ST_MTIME]
+            directory = ServerDirectory(fname=fname)
+            try:
+                directory = ServerDirectory(f.read())
+            except ConfigError:
+                LOG.warn("Ignoring invalid directory (!)")
+                continue
+            f.close()
+            for s in directory.getServers():
+                self.serverList.append((s, 'D'))
+            break
+
+        serverDir = os.path.join(self.dir, "servers")
+        createPrivateDir(serverDir)
+        for fn in os.listdir(serverDir):
             # Try to read a file: is it a server descriptor?
-            p = os.path.join(self.directory, f)
+            p = os.path.join(self.directory, fn)
             try:
                 info = ServerInfo(fname=p, assumeValid=0)
             except ConfigError:
                 LOG.warn("Invalid server descriptor %s", p)
                 continue
+            mtime = os.stat(p)[stat.ST_MTIME]
+            if mtime > self.lastModified:
+                self.lastModifed = mtime
+            self.serverList.append((info, "I:%s"%fn))
+        self.__save()
+        self.__scanning = 1
+        self.__load()
+    def __load(self):
+        "DOCDOC"
+        #DOCDOC
+        try:
+            f = open(os.path.join(self.dir, "cache"), 'rb')
+            cached = cPickle.load(f)
+            self.lastModified, self.lastDowload, self.serverList = cached
+            f.close()
+            self.__rebuildTables()
+            return
+        except OSError, e:
+            LOG.info("Couldn't create server cache: %s", e)
+        except (cPickle.UnpicklingError, ValueError), e:
+            LOG.info("Couldn't unpickle server cache: %s", e)
+        if self.__scanning:
+            raise MixFatalError("Recursive error while regenerating cache")
+        self.rescan()
+    def __save(self):
+        "DOCDOC"
+        fname = os.path.join(self.dir, "cache.new")
+        os.unlink(fname)
+        f = open(fname, 'wb')
+        cPickle.dump((self.lastModified, self.lastDownload, self.serverList),
+                     f, 1)
+        f.close()
+        os.rename(fname, os.path.join(self.dir, "cache"))
+    def importFromFile(self, filename):
+        "DOCDOC"
+        #DOCDOC
+        f = open(filename, 'r')
+        contents = f.read()
+        f.close()
+        info = ServerInfo(string=contents)
 
-            # Find its nickname and normalized filename
-            serverSection = info['Server']
-            nickname = serverSection['Nickname']
+        nickname = info.getNickname()
+        identity = info.getIdentity()
+        for s, _ in self.serverList:
+            if s.getNickname() == nickname:
+                if not pk_same_public_key(identity, s.getIdentity()):
+                    raise MixError("Identity key changed for server %s in %s",
+                                   nickname, filename)
+        
+        fnshort = "%s-%s"%(nickname, formatFnameTime())
+        fname = os.path.join(self.dir, "servers", fnshort)
+        f = open(fname, 'w')
+        f.write(contents)
+        f.close()
+        self.serverList.append((info, 'I:%s', fnshort))
+        self.__save()
+        self.__rebuildTables()
+    def expungeByNickname(self, nickname):
+        "DOCDOC"
+        #DOCDOC
+        n = 0
+        newList = []
+        for info, source in self.serverList:
+            if source == 'D' or info.getNickname() != nickname:
+                newList.append((info, source))
+                continue
+            n += 1
+            try:
+                os.unlink(os.path.join(self.dir, "servers", fn))
+            except OSError, e:
+                Log.error("Couldn't remove %s", fn)
 
-            if '.' in f:
-                f = f[:f.rindex('.')]
+        self.serverList = newList
+        if n:
+            self.lastModifed = time.time()
+            self.__save()
+        return n
 
-            # Skip the descriptor if it isn't valid yet...
-            if now < serverSection['Valid-After']:
-                LOG.info("Ignoring future decriptor %s", p)
-                continue
-            # ... or if it's expired ...
-            if now >= serverSection['Valid-Until']:
-                LOG.info("Ignoring expired decriptor %s", p)
-                continue
-            # ... or if it's going to expire within 3 hours (HACK!).
-            if now + 3*60*60 >= serverSection['Valid-Until']:
-                LOG.info("Ignoring soon-to-expire decriptor %s", p)
-                continue
-            # Only allow one server per nickname ...
-            if self.byNickname.has_key(nickname):
-                LOG.warn(
-                    "Ignoring descriptor %s with duplicate nickname %s",
-                    p, nickname)
-                continue
-            # ... and per normalized filename.
-            if self.byFilename.has_key(f):
-                LOG.warn(
-                    "Ignoring descriptor %s with duplicate prefix %s",
-                    p, f)
+    def __rebuildTables(self):
+        "DOCDOC"
+        #DOCDOC
+        self.byNickname = {}
+        self.allServers = []
+        self.byCapability = { 'mbox': [], 
+                              'smtp': [],
+                              'relay': [],
+                              None: self.allServers }
+        for info, where in self.serverList:
+            nn = info.getNickname()
+            lists = [ self.allServers, self.byNickname.setdefault(nn, []) ]
+            for c in info.getCaps():
+                lists.append( self.byCapability[c] )
+            for lst in lists:
+                lst.append((info, where))
+
+    def listServers(self):
+        """Returns a linewise listing of the current servers and their caps.
+           stdout.  This will go away or get refactored in future versions
+           once we have client-level modules."""
+        #DOCDOC
+        lines = []
+        nicknames = self.byNickname.keys()
+        nicknames.sort()
+        longestnamelen = max(map(len, nicknames))
+        fmtlen = min(longestnamelen, 20)
+        format = "%"+str(fmtlen)+"s:"
+        for n in nicknames:
+            lines.append(format%n)
+            for info, where in self.byNickname[n]:
+                caps = info.getCaps()
+                va = formatDate(info['Server']['Valid-After'])
+                vu = formatDate(info['Server']['Valid-Until'])
+                line = "   %15s (valid %s to %s)"%(" ".join(caps),va,vu)
+                lines.append(line)
+        return lines
+
+    def __findOne(self, lst, startAt, endAt):
+        "DOCDOC"
+        res = self.__find(lst, startAt, endAt)
+        if res:
+            return res[0]
+        return None
+
+    def __find(self, lst, startAt, endAt):
+        "DOCDOC"
+        lst = [ info for info, _ in lst if info.isValidFrom(startAt, endAt) ]
+        # XXXX This is not really good: servers may be the same, even if
+        # XXXX their nicknames are different.  The logic should probably
+        # XXXX go into directory, though.
+        u = {}
+        for info in lst:
+            n = info.getNickname()
+            if u.has_key(n):
+                if info.isExpiredAt(u[n]['Server']['Valid-Until']):
+                    continue
+            u[n] = info
+
+        return u.values()
+
+    def clean(self, now=None):
+        "DOCDOC"
+        #DOCDOC
+        if now is None:
+            now = time.time()
+        cutoff = now - 600
+
+        newServers = []
+        for info, where in self.serverList:
+            if where == 'D':
+                newServers.append((info, where))
                 continue
-            LOG.info("Loaded server %s from %s", nickname, f)
-            # Okay, it's good. Cache it.
-            self.byNickname[nickname] = info
-            self.byFilename[f] = info
+            elif info.isExpiredAt(cutoff):
+                pass
+            else:
+                valid = info.getIntervalSet()
+                for s in self.byNickname[info.getNickname()]:
+                    if s.isNewerThan(info):
+                        valid -= s.getIntervalSet()
+                if not valid.isEmpty():
+                    newServers.append((info, where))
+                    continue
+            try:
+                os.unlink(os.path.join(self.dir, "servers", where[2:]))
+            except OSError, e:
+                LOG.info("Couldn't remove %s: %s", where[2:], e)
+                    
+        self.serverList = newServers
+        self.__rebuildTables()
+
+    def getServerInfo(self, name, startAt=None, endAt=None, strict=0):
+        "DOCDOC"
+        #DOCDOC
+        if startAt is None:
+            startAt = time.time()
+        if endAt is None:
+            endAt = startAt + 3600
 
-    def getServerInfo(self, name):
-        """Return a ServerInfo object corresponding to 'name'.  If 'name' is
-           a ServerInfo object, returns 'name'.  Otherwise, checks server by
-           nickname, then by filename within the keystore, then by filename
-           on the file system. If no server is found, returns None."""
         if isinstance(name, ServerInfo):
             return name
         elif self.byNickname.has_key(name):
-            return self.byNickname[name]
-        elif self.byFilename.has_key(name):
-            return self.byFilename[name]
+            s = self.__find(self.byNickname[name], startAt, endAt)
         elif os.path.exists(name):
             try:
                 return ServerInfo(fname=name, assumeValid=0)
@@ -138,60 +341,300 @@
             except ConfigError, e:
                 raise MixError("Couldn't parse descriptor %s: %s" %
                                (name, e))
+        elif strict:
+            raise MixError("Couldn't find descriptor %s")
         else:
             return None
 
-    def getPath(self, serverList):
-        """Given a sequence of strings of ServerInfo objects, resolves each
-           one according to the rule of getServerInfo, and returns a list of
-           ServerInfos.  Raises MixError if any server can't be resolved."""
-        path = []
-        for s in serverList:
-            if isinstance(s, ServerInfo):
-                path.append(s)
-            elif isinstance(s, types.StringType):
-                server = self.getServerInfo(s)
-                if server is not None:
-                    path.append(server)
-                else:
-                    raise MixError("Couldn't find descriptor %s" % s)
-        return path
+    def getPath(self, midCap=None, endCap=None, length=None,
+                startServers=(), endServers=(),
+                startAt=None, endAt=None, prng=None):
+        "DOCDOC"
+        #DOCDOC
+        if startAt is None:
+            startAt = time.time()
+        if endAt is None:
+            endAt = startAt + 3600
+        if prng is None:
+            prng = mixminion.Crypto.getCommonPRNG()
+        
+        startServers = [ self.getServerInfo(name,startAt,endAt,1) 
+                         for name in startServers ]
+        endServers = [ self.getServerInfo(name,startAt,endAt,1)
+                       for name in endServers ]
+        nNeeded = 0
+        if length:
+            nNeeded = length - len(startServers) - len(endServers)
+        
+        if nNeeded <= 0:
+            return startServers + endServers
 
-    def listServers(self):
-        """Returns a linewise listing of the current servers and their caps.
-           stdout.  This will go away or get refactored in future versions
-           once we have real path selection and client-level modules."""
-        lines = []
-        nicknames = self.byNickname.keys()
-        nicknames.sort()
-        longestnamelen = max(map(len, nicknames))
-        fmtlen = min(longestnamelen, 20)
-        format = "%"+str(fmtlen)+"s (expires %s): %s"
-        for n in nicknames:
-            caps = []
-            si = self.byNickname[n]
-            if si['Delivery/MBOX'].get('Version',None):
-                caps.append("mbox")
-            if si['Delivery/SMTP'].get('Version',None):
-                caps.append("smtp")
-            # XXXX This next check is highly bogus.
-            if (si['Incoming/MMTP'].get('Version',None) and 
-                si['Outgoing/MMTP'].get('Version',None)):
-                caps.append("relay")
-            until = formatDate(si['Server']['Valid-Until'])
-            line = format % (n, until, " ".join(caps))
-            lines.append(line)
-        return lines
+        endList = self.__find(self.byCapability[endCap],startAt,endAt)
+        if not endServers:
+            if not endList:
+                raise MixError("No %s servers known"% endCap)
+            LOG.info("Choosing from among %s %s servers",
+                     len(endList), endCap)
+            endServers = [ self.prng.pick(endList) ]
+            LOG.debug("Chose %s", endServers[0].getNickname())
+            nNeeded -= 1
 
-    def getRandomServers(self, prng, n):
-        """Returns a list of n different servers, in random order, according
-           to prng.  Raises MixError if not enough exist.
+        if nNeeded == 0:
+            return startServers + endServers
 
-           (This isn't actually used.)"""
-        vals = self.byNickname.values()
-        if len(vals) < n:
-            raise MixError("Not enough servers (%s requested)", n)
-        return prng.shuffle(vals, n)
+        # This is hard.  We need to find a number of relay servers for
+        # the midddle of the list.  If len(midList) >> length, we should
+        # remove all servers that already appear, and shuffle from the
+        # rest.  Otherwise, if len(midList) >= 3, we pick one-by-one from 
+        # the list of possibilities, just making sure not to get 2 in a row.
+        # Otherwise, len(midList) <= 3, so we just wing it.
+        #
+        # FFFF This algorithm is far from ideal, but the answer is to
+        # FFFF get more servers deployed.
+        midList = self.__find(self.byCapability[midCap],startAt,endAt)
+        used = [ info.getNickname() 
+                 for info in list(startServers)+list(endServers) ]
+        unusedMidList = [ info for info in midList 
+                          if info.getNickname() not in used ]
+        if len(unusedMidList) >= length*1.1:
+            midServers = prng.shuffle(unusedMidList, nNeeded)
+        elif len(midList) >= 3:
+            LOG.warn("Not enough servers for distinct path (only %s unused)",
+                     len(unusedMidList))
+            midServers = []
+            if startServers:
+                prevNickname = startServers[-1].getNickname()
+            else:
+                prevNickname = " (impossible nickname) "
+            if endServers:
+                endNickname = endServers[0].getNickname()
+            else:
+                endNickname = " (impossible nickname) "
+
+            while nNeeded: 
+                info = prng.pick(midList)
+                n = info.getNickname()
+                if n != prevNickname and (nNeeded > 1 or n != endNickname):
+                    midServers.append(info)
+                    prevNickname = n
+                    nNeeded -= 1
+        elif midList == 2:
+            LOG.warn("Not enough relays to avoid same-server hops")
+            midList = prng.shuffle(midList)
+            midServers = (midList * ceilDiv(nNeeded, 2))[-nNeeded]
+        elif midList == 1:
+            LOG.warn("Only one relay known")
+            midServers = midList
+        else:
+            raise MixError("No relays known")
+            
+        LOG.info("Chose path: [%s][%s][%s]",
+                 " ".join([s.getNickname() for n in startServers]),
+                 " ".join([s.getNickname() for n in midServers]),
+                 " ".join([s.getNickname() for n in endServers]))
+
+        return startServers + midServers + endServers
+
+def resolvePath(keystore, address, path1, path2, enterPath, exitPath,
+                nHops, nSwap, startAt=None, endAt=None):
+    "DOCDOC"
+    #DOCDOC
+    if startAt is None:
+        startAt = time.time()
+    if endAt is None:
+        endAt = time.time()+3*60*60 # FFFF Configurable
+
+    routingType, _, exitNode = address.getRouting()
+    if exitNode:
+        exitNode = keystore.getServerInfo(exitNode, startAt, endAt)
+
+    if routingType == MBOX_TYPE:
+        exitCap = 'mbox'
+    elif routingType == SMTP_TYPE:
+        exitCap = 'smtp'
+    else:
+        exitCap = None
+ 
+    if path1 and path2:
+        path = path1+path2
+        path = keystore.getPath(length=len(path), startServers=path,
+                                startAt=startAt, endAt=endAt)
+        if exitNode is not None:
+            path.append(exitNode)
+        nSwap = len(path1)-1
+    elif path1 or path2:
+        raise MixError("--path1 and --path2 must both be specified or not")
+    else:
+        if exitNode is not None:
+            exitPath.append(exitNode)
+        nHops = nHops - len(enterPath) - len(exitPath)
+        path = keystore.getPath(length=nHops,
+                                startServers=enterPath,
+                                endServers=exitPath,
+                                midCap='relay', endCap=exitCap,
+                                startAt=startAt, endAt=endAt)
+        if nSwap < 0:
+            nSwap = ceilDiv(len(path),2)
+
+    for server in path[:-1]:
+        if "relay" not in server.getCaps():
+            raise MixError("Server %s does not support relay"
+                           % server.getNickname())
+    if exitCap and exitCap not in path[-1].getCaps():
+        raise MixError("Server %s does not support %s"
+                       % (server.getNickname(), exitCap))
+    
+    return path[:nSwap+1], path[nSwap+1:]
+
+## class TrivialKeystore:
+##     """This is a temporary keystore implementation until we get a working
+##        directory server implementation.
+
+##        The idea couldn't be simpler: we just keep a directory of files, each
+##        containing a single server descriptor.  We cache nothing; we validate
+##        everything; we have no automatic path generation.  Servers can be
+##        accessed by nickname, by filename within our directory, or by filename
+##        from elsewhere on the filesystem.
+
+##        We skip all server descriptors that have expired, that will
+##        soon expire, or which aren't yet in effect.
+##        """
+##     ## Fields:
+##     # directory: path to the directory we scan for server descriptors.
+##     # byNickname: a map from nickname to valid ServerInfo object.
+##     # byFilename: a map from filename within self.directory to valid
+##     #     ServerInfo object.
+##     def __init__(self, directory, now=None):
+##         """Create a new TrivialKeystore to access the descriptors stored in
+##            directory.  Selects descriptors that are valid at the time 'now',
+##            or at the current time if 'now' is None."""
+##         self.directory = directory
+##         createPrivateDir(directory)
+##         self.byNickname = {}
+##         self.byFilename = {}
+
+##         if now is None:
+##             now = time.time()
+
+##         for f in os.listdir(self.directory):
+##             # Try to read a file: is it a server descriptor?
+##             p = os.path.join(self.directory, f)
+##             try:
+##                 info = ServerInfo(fname=p, assumeValid=0)
+##             except ConfigError:
+##                 LOG.warn("Invalid server descriptor %s", p)
+##                 continue
+
+##             # Find its nickname and normalized filename
+##             serverSection = info['Server']
+##             nickname = serverSection['Nickname']
+
+##             if '.' in f:
+##                 f = f[:f.rindex('.')]
+
+##             # Skip the descriptor if it isn't valid yet...
+##             if now < serverSection['Valid-After']:
+##                 LOG.info("Ignoring future decriptor %s", p)
+##                 continue
+##             # ... or if it's expired ...
+##             if now >= serverSection['Valid-Until']:
+##                 LOG.info("Ignoring expired decriptor %s", p)
+##                 continue
+##             # ... or if it's going to expire within 3 hours (HACK!).
+##             if now + 3*60*60 >= serverSection['Valid-Until']:
+##                 LOG.info("Ignoring soon-to-expire decriptor %s", p)
+##                 continue
+##             # Only allow one server per nickname ...
+##             if self.byNickname.has_key(nickname):
+##                 LOG.warn(
+##                     "Ignoring descriptor %s with duplicate nickname %s",
+##                     p, nickname)
+##                 continue
+##             # ... and per normalized filename.
+##             if self.byFilename.has_key(f):
+##                 LOG.warn(
+##                     "Ignoring descriptor %s with duplicate prefix %s",
+##                     p, f)
+##                 continue
+##             LOG.info("Loaded server %s from %s", nickname, f)
+##             # Okay, it's good. Cache it.
+##             self.byNickname[nickname] = info
+##             self.byFilename[f] = info
+
+##     def getServerInfo(self, name):
+##         """Return a ServerInfo object corresponding to 'name'.  If 'name' is
+##            a ServerInfo object, returns 'name'.  Otherwise, checks server by
+##            nickname, then by filename within the keystore, then by filename
+##            on the file system. If no server is found, returns None."""
+##         if isinstance(name, ServerInfo):
+##             return name
+##         elif self.byNickname.has_key(name):
+##             return self.byNickname[name]
+##         elif self.byFilename.has_key(name):
+##             return self.byFilename[name]
+##         elif os.path.exists(name):
+##             try:
+##                 return ServerInfo(fname=name, assumeValid=0)
+##             except OSError, e:
+##                 raise MixError("Couldn't read descriptor %s: %s" %
+##                                (name, e))
+##             except ConfigError, e:
+##                 raise MixError("Couldn't parse descriptor %s: %s" %
+##                                (name, e))
+##         else:
+##             return None
+
+##     def getPath(self, serverList):
+##         """Given a sequence of strings of ServerInfo objects, resolves each
+##            one according to the rule of getServerInfo, and returns a list of
+##            ServerInfos.  Raises MixError if any server can't be resolved."""
+##         path = []
+##         for s in serverList:
+##             if isinstance(s, ServerInfo):
+##                 path.append(s)
+##             elif isinstance(s, types.StringType):
+##                 server = self.getServerInfo(s)
+##                 if server is not None:
+##                     path.append(server)
+##                 else:
+##                     raise MixError("Couldn't find descriptor %s" % s)
+##         return path
+
+##     def listServers(self):
+##         """Returns a linewise listing of the current servers and their caps.
+##            stdout.  This will go away or get refactored in future versions
+##            once we have real path selection and client-level modules."""
+##         lines = []
+##         nicknames = self.byNickname.keys()
+##         nicknames.sort()
+##         longestnamelen = max(map(len, nicknames))
+##         fmtlen = min(longestnamelen, 20)
+##         format = "%"+str(fmtlen)+"s (expires %s): %s"
+##         for n in nicknames:
+##             caps = []
+##             si = self.byNickname[n]
+##             if si['Delivery/MBOX'].get('Version',None):
+##                 caps.append("mbox")
+##             if si['Delivery/SMTP'].get('Version',None):
+##                 caps.append("smtp")
+##             # XXXX This next check is highly bogus.
+##             if (si['Incoming/MMTP'].get('Version',None) and 
+##                 si['Outgoing/MMTP'].get('Version',None)):
+##                 caps.append("relay")
+##             until = formatDate(si['Server']['Valid-Until'])
+##             line = format % (n, until, " ".join(caps))
+##             lines.append(line)
+##         return lines
+
+##     def getRandomServers(self, prng, n):
+##         """Returns a list of n different servers, in random order, according
+##            to prng.  Raises MixError if not enough exist.
+
+##            (This isn't actually used.)"""
+##         vals = self.byNickname.values()
+##         if len(vals) < n:
+##             raise MixError("Not enough servers (%s requested)", n)
+##         return prng.shuffle(vals, n)
 
 def installDefaultConfig(fname):
     """Create a default, 'fail-safe' configuration in a given file"""
@@ -229,7 +672,6 @@
        to generating and sending forward messages"""
     ## Fields:
     # config: The ClientConfig object with the current configuration
-    # keystore: A TrivialKeystore object
     # prng: A pseudo-random number generator for padding and path selection
     def __init__(self, conf):
         """Create a new MixminionClient with a given configuration"""
@@ -241,57 +683,31 @@
         #createPrivateDir(os.path.join(userdir, 'surbs'))
         createPrivateDir(os.path.join(userdir, 'servers'))
 
-        # Get directory cache
-        self.keystore = TrivialKeystore(
-            os.path.join(userdir,"servers"))
-
         # Initialize PRNG
         self.prng = mixminion.Crypto.getCommonPRNG()
 
-    def sendForwardMessage(self, address, payload, path1, path2):
+    def sendForwardMessage(self, address, payload, servers1, servers2):
         """Generate and send a forward message.
             address -- the results of a parseAddress call
             payload -- the contents of the message to send
-            path1,path2 -- lists of servers or server names for the first and
-               second legs of the path, respectively.  These are processed
-               as described in TrivialKeystore.getServerInfo"""
+            path1,path2 -- lists of servers for the first and second legs of
+               the path, respectively."""
+
         message, firstHop = \
-                 self.generateForwardMessage(address, payload, path1, path2)
+                 self.generateForwardMessage(address, payload, 
+                                             servers1, servers2)
 
         self.sendMessages([message], firstHop)
 
-    def generateForwardMessage(self, address, payload, path1, path2):
+    def generateForwardMessage(self, address, payload, servers1, servers2):
         """Generate a forward message, but do not send it.  Returns
            a tuple of (the message body, a ServerInfo for the first hop.)
 
             address -- the results of a parseAddress call
             payload -- the contents of the message to send
-            path1,path2 -- lists of servers or server names for the first and
-               second legs of the path, respectively.  These are processed
-               as described in TrivialKeystore.getServerInfo"""
-        if not path1:
-            raise MixError("No servers in first leg of path")
-        if not path2:
-            raise MixError("No servers in second leg of path")
-
-        servers1 = self.keystore.getPath(path1)
-        servers2 = self.keystore.getPath(path2)
+            path1,path2 -- lists of servers."""
 
-        routingType, routingInfo, lastHop = address.getRouting()
-        if lastHop is None:
-            lastServer = servers2[-1]
-            # FFFF This is only a temporary solution.  It needs to get
-            # FFFF rethought, or refactored into ServerInfo, or something.
-            if routingType == SMTP_TYPE:
-                ok = lastServer['Delivery/SMTP'].get('Version',None)
-                if not ok:
-                    raise MixError("Last hop doesn't support SMTP")
-            elif routingType == MBOX_TYPE:
-                ok = lastServer['Delivery/MBOX'].get('Version',None)
-                if not ok:
-                    raise MixError("Last hop doesn't support MBOX")
-        else:
-            servers2.append(self.keystore.getServerInfo(lastHop))
+        routingType, routingInfo, _ = address.getRouting()
         msg = mixminion.BuildMessage.buildForwardMessage(
             payload, routingType, routingInfo, servers1, servers2,
             self.prng)
@@ -368,6 +784,14 @@
         return self.exitType, self.exitAddress, self.lastHop
 
 def readConfigFile(configFile):
+    "DOCDOC"
+    if configFile is None:
+        configFile = os.environ.get("MIXMINIONRC", None)
+    if configFile is None:
+        configFile = "~/.mixminionrc"
+    configFile = os.path.expanduser(configFile)
+    if not os.path.exists(configFile):
+        installDefaultConfig(configFile)    
     try:
         return ClientConfig(fname=configFile)
     except (IOError, OSError), e:
@@ -380,22 +804,38 @@
         sys.exit(1)
     return None #suppress pychecker warning
 
+def usageAndExit(cmd, error=None):
+    #XXXX002 correct this.
+    if error:
+        print >>stderr, "ERROR: %s"%error
+    print >>sys.stderr, """\
+Usage: %s [-h] [-v] [-f configfile] [-i inputfile]
+          [--path1=server1,server2,...]
+          [--path2=server1,server2,...] [-t <address>]"""%cmd
+    sys.exit(1)
+
 # NOTE: This isn't anything LIKE the final client interface.  Many or all
 #       options will change between now and 1.0.0
 def runClient(cmd, args):
-    options, args = getopt.getopt(args, "hvf:i:t:",
+    options, args = getopt.getopt(args, "hvf:i:t:H:",
                                   ["help", "verbose", "config=", "input=",
-                                   "path1=", "path2=", "to="])
+                                   "path1=", "path2=", "to=", "hops=",
+                                   "swap-at=", "enter=", "exit=",
+                                  ])
     configFile = '~/.mixminionrc'
     usage = 0
     inFile = "-"
     verbose = 0
     path1 = []
     path2 = []
+    enter = []
+    exit = []
+    swapAt = -1
+    hops = -1 # XXXX Make configurable
     address = None
     for opt,val in options:
         if opt in ('-h', '--help'):
-            usage=1
+            usageAndExit(cmd)
         elif opt in ('-f', '--config'):
             configFile = val
         elif opt in ('-i', '--input'):
@@ -406,37 +846,19 @@
             path1.extend(val.split(","))
         elif opt == '--path2':
             path2.extend(val.split(","))
+        elif opt in ('-H', '--hops'):
+            try:
+                hops = int(val)
+            except ValueError:
+                usageAndExit(cmd, "%s expects an integer"%opt)
         elif opt in ('-t', '--to'):
             address = parseAddress(val)
     if args:
-        print >>sys.stderr, "Unexpected options."
-        usage = 1
-    if not path1:
-        print >>sys.stderr, "First leg of path was not specified"
-        usage = 1
-    if not path2:
-        print >>sys.stderr, "Second leg of path was not specified"
-        usage = 1
+        usageEndExit(cmd,"Unexpected options")
     if address is None:
-        print >>sys.stderr, "No recipient specified"
-        usage = 1
-    if usage:
-        print >>sys.stderr, """\
-Usage: %s [-h] [-v] [-f configfile] [-i inputfile]
-          [--path1=server1,server2,...]
-          [--path2=server1,server2,...] [-t <address>]"""%cmd
-        sys.exit(1)
-
-    if configFile is None:
-        configFile = os.environ.get("MIXMINIONRC", None)
-        if configFile is None:
-            configFile = "~/.mixminionrc"
+        usageAndExit(cmd,"No recipient specified")
 
-    configFile = os.path.expanduser(configFile)
-    if not os.path.exists(configFile):
-        installDefaultConfig(configFile)
     config = readConfigFile(configFile)
-
     LOG.configure(config)
     if verbose:
         LOG.setMinSeverity("DEBUG")
@@ -445,6 +867,17 @@
     mixminion.Common.configureShredCommand(config)
     mixminion.Crypto.init_crypto(config)
 
+    keystore = ClientKeystore(os.path.expanduser(config['User']['UserDir']))
+    #try:
+    if 1:
+        path1, path2 = resolvePath(keystore, address,
+                                   path1, path2,
+                                   enterPath, exitPath,
+                                   nHops, nSwap)
+    #except MixError, e:
+    #    print e
+    #    sys.exit(1)
+
     client = MixminionClient(config)
 
     if inFile == '-':
@@ -466,16 +899,7 @@
         elif o in ('-f', '--config'):
             configFile = v
 
-    # XXXX duplicate code; refactor into separate method.
-    if configFile is None:
-        configFile = os.environ.get("MIXMINIONRC", None)
-        if configFile is None:
-            configFile = "~/.mixminionrc"
-    configFile = os.path.expanduser(configFile)
-    if not os.path.exists(configFile):
-        installDefaultConfig(configFile)
     config = readConfigFile(configFile)
-
     userdir = os.path.expanduser(config['User']['UserDir'])
     createPrivateDir(os.path.join(userdir, 'servers'))
 

Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- Config.py	29 Dec 2002 20:46:54 -0000	1.28
+++ Config.py	3 Jan 2003 05:14:47 -0000	1.29
@@ -53,6 +53,7 @@
 
 import calendar
 import binascii
+import gzip
 import os
 import re
 import socket # for inet_aton and error
@@ -60,7 +61,8 @@
 
 import mixminion.Common
 import mixminion.Crypto
-from mixminion.Common import MixError, LOG, isPrintingAscii, stripSpace
+from mixminion.Common import MixError, LOG, isPrintingAscii, stripSpace, \
+     _ALLCHARS
 
 class ConfigError(MixError):
     """Thrown when an error is found in a configuration file."""
@@ -287,6 +289,25 @@
 
     return calendar.timegm((yyyy,MM,dd,hh,mm,ss,0,0,0))
 
+_NICKNAME_CHARS = ("ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
+                   "abcdefghijklmnopqrstuvwxyz"+
+                   "0123456789_.!@#-")
+MAX_NICKNAME = 128
+def _parseNickname(s):
+    """Validation function.  Returns true iff s contains a valoid
+       server nickname-- that is, a string of 1..128 characters,
+       containing only the characters [A-Za-z0-9_.!@#] and '-'.
+       """
+    s = s.strip()
+    bad = s.translate(_ALLCHARS, _NICKNAME_CHARS)
+    if len(bad):
+        raise ConfigError("Invalid characters %r in nickname %r" % (bad,s))
+    if len(s) > MAX_NICKNAME:
+        raise ConfigError("Nickname is too long")
+    elif len(s) == 0:
+        raise ConfigError("Nickname is too short")
+    return s
+
 #----------------------------------------------------------------------
 
 # Regular expression to match a section header.
@@ -454,6 +475,9 @@
         """Create a new _ConfigFile.  If <filename> is set, read from
            a corresponding file.  If <string> is set, parse its contents.
 
+           (If <filename> ends with ".gz", assume a file compressed
+           with gzip.)
+
            If <assumeValid> is true, skip all unnecessary validation
            steps.  (Use this to load a file that's already been checked as
            valid.)"""
@@ -482,7 +506,11 @@
            the contents of this object unchanged."""
         if not self.fname:
             return
-        f = open(self.fname, 'r')
+        if self.fname.endswith(".gz"):
+            #XXXX002 test!
+            f = gzip.GzipFile(self.fname, 'r')
+        else:
+            f = open(self.fname, 'r')
         try:
             self.__reload(f, None)
         finally:

Index: Crypto.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Crypto.py,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- Crypto.py	31 Dec 2002 04:48:46 -0000	1.31
+++ Crypto.py	3 Jan 2003 05:14:47 -0000	1.32
@@ -12,6 +12,7 @@
 import os
 import stat
 import sys
+import binascii
 from types import StringType
 
 import mixminion._minionlib as _ml
@@ -22,11 +23,11 @@
             'lioness_decrypt', 'lioness_encrypt', 'openssl_seed',
             'pk_check_signature', 'pk_decode_private_key',
             'pk_decode_public_key', 'pk_decrypt', 'pk_encode_private_key',
-            'pk_encode_public_key', 'pk_encrypt', 'pk_from_modulus',
-            'pk_generate', 'pk_get_modulus', 'pk_sign', 'prng', 'sha1',
-            'strxor', 'trng', 'AES_KEY_LEN', 'DIGEST_LEN',
-            'HEADER_SECRET_MODE', 'PRNG_MODE', 'RANDOM_JUNK_MODE',
-            'HEADER_ENCRYPT_MODE', 'APPLICATION_KEY_MODE',
+            'pk_encode_public_key', 'pk_encrypt', 'pk_fingerprint',
+            'pk_from_modulus', 'pk_generate', 'pk_get_modulus',
+            'pk_same_public_key', 'pk_sign', 'prng', 'sha1', 'strxor', 'trng',
+            'AES_KEY_LEN', 'DIGEST_LEN', 'HEADER_SECRET_MODE', 'PRNG_MODE',
+            'RANDOM_JUNK_MODE', 'HEADER_ENCRYPT_MODE', 'APPLICATION_KEY_MODE',
             'PAYLOAD_ENCRYPT_MODE', 'HIDE_HEADER_MODE' ]
 
 # Expose _minionlib.CryptoError as Crypto.CryptoError
@@ -258,9 +259,14 @@
 
 def pk_same_public_key(key1, key2):
     """Return true iff key1 and key2 are the same key."""
-    #XXXX TEST
     return key1.encode_key(1) == key2.encode_key(1)
 
+def pk_fingerprint(key):
+    """Return the 40-character fingerprint of public key 'key'.  This
+       is computed as the hex encoding of the SHA-1 digest of the
+       ASN.1 encoding of the public portion of key."""
+    return binascii.b2a_hex(sha1(key.encode_key(1))).upper()
+
 def pk_PEM_save(rsa, filename, password=None):
     """Save a PEM-encoded private key to a file.  If <password> is provided,
        encrypt the key using the password."""
@@ -487,6 +493,11 @@
             res = self.bytes[:n]
             self.bytes = self.bytes[n:]
             return res
+
+    def pick(self, lst):
+        "DOCDOC"
+        #XXXX002 test
+        return lst[self.getInt(len(lst))]
 
     def shuffle(self, lst, n=None):
         """Rearranges the elements of lst so that the first n elements

Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- Main.py	16 Dec 2002 02:40:11 -0000	1.15
+++ Main.py	3 Jan 2003 05:14:47 -0000	1.16
@@ -120,6 +120,7 @@
     "server" :         ( 'mixminion.server.ServerMain', 'runServer' ),
     "server-keygen" :  ( 'mixminion.server.ServerMain', 'runKeygen'),
     "server-DELKEYS" : ( 'mixminion.server.ServerMain', 'removeKeys'),
+    "dir":             ( 'mixminion.directory.DirMain', 'main'),
 }
 
 def printVersion(cmd,args):

Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- ServerInfo.py	31 Dec 2002 04:48:46 -0000	1.28
+++ ServerInfo.py	3 Jan 2003 05:14:47 -0000	1.29
@@ -8,22 +8,21 @@
    descriptors.
    """
 
-__all__ = [ 'ServerInfo' ]
+__all__ = [ 'ServerInfo', 'ServerDirectory' ]
 
+import gzip
 import re
 import time
 
 import mixminion.Config
 import mixminion.Crypto
 
-from mixminion.Common import LOG, MixError, createPrivateDir, formatBase64, \
-    formatDate, formatTime
+from mixminion.Common import IntervalSet, LOG, MixError, createPrivateDir, \
+    formatBase64, formatDate, formatTime
 from mixminion.Config import ConfigError
 from mixminion.Packet import IPV4Info
-from mixminion.Crypto import DIGEST_LEN
+from mixminion.Crypto import CryptoError, DIGEST_LEN, pk_check_signature
 
-# Longest allowed Nickname
-MAX_NICKNAME = 128
 # Longest allowed Contact email
 MAX_CONTACT = 256
 # Longest allowed Comments field
@@ -38,15 +37,15 @@
 # tmp alias to make this easier to spell.
 C = mixminion.Config
 class ServerInfo(mixminion.Config._ConfigFile):
-    ## Fields
-    # isValidated: DOCDOC
+    ## Fields: (as in ConfigFile, plus)
+    # _isValidated: flag.  Has this serverInfo been fully validated?
     """A ServerInfo object holds a parsed server descriptor."""
     _restrictFormat = 1
     _syntax = {
         "Server" : { "__SECTION__": ("REQUIRE", None, None),
                      "Descriptor-Version": ("REQUIRE", None, None),
                      "IP": ("REQUIRE", C._parseIP, None),
-                     "Nickname": ("REQUIRE", None, None),
+                     "Nickname": ("REQUIRE", C._parseNickname, None),
                      "Identity": ("REQUIRE", C._parsePublicKey, None),
                      "Digest": ("REQUIRE", C._parseBase64, None),
                      "Signature": ("REQUIRE", C._parseBase64, None),
@@ -93,8 +92,6 @@
         if server['Descriptor-Version'] != '0.1':
             raise ConfigError("Unrecognized descriptor version %r",
                               server['Descriptor-Version'])
-        if len(server['Nickname']) > MAX_NICKNAME:
-            raise ConfigError("Nickname too long")
         identityKey = server['Identity']
         identityBytes = identityKey.get_modulus_bytes()
         if not (MIN_IDENTITY_BYTES <= identityBytes <= MAX_IDENTITY_BYTES):
@@ -118,9 +115,13 @@
             raise ConfigError("Invalid digest")
 
         # Check signature
-        if digest != mixminion.Crypto.pk_check_signature(server['Signature'],
-                                                         identityKey):
+        try:
+            signedDigest = pk_check_signature(server['Signature'], identityKey)
+        except CryptoError:
             raise ConfigError("Invalid signature")
+        
+        if digest != signedDigest:
+            raise ConfigError("Signed digest is incorrect")
 
         ## Incoming/MMTP section
         inMMTP = sections['Incoming/MMTP']
@@ -169,9 +170,160 @@
            to this server."""
         return IPV4Info(self.getAddr(), self.getPort(), self.getKeyID())
 
+    def getIdentity(self):
+        return self['Server']['Identity']
+
+    def getCaps(self):
+        # FFFF refactor this once we have client addresses.
+        caps = []
+        if not self['Incoming/MMTP'].get('Version',None):
+            return caps
+        if self['Delivery/MBOX'].get('Version', None):
+            caps.append('mbox')
+        if self['Delivery/SMTP'].get('Version', None):
+            caps.append('smtp')
+        # XXXX This next check is highly bogus.
+        if self['Outgoing/MMTP'].get('Version',None):
+            caps.append('relay')
+        return caps
+
     def isValidated(self):
-        "DOCDOC"
+        """Return true iff this ServerInfo has been validated"""
         return self._isValidated
+
+    def getIntervalSet(self):
+        """Return an IntervalSet covering all the time at which this
+           ServerInfo is valid."""
+        #XXXX002 test
+        return IntervalSet([(self['Server']['Valid-After'],
+                             self['Server']['Valid-Until'])])
+
+    def isExpiredAt(self, when):
+        """Return true iff this ServerInfo expires before time 'when'."""
+        #XXXX002 test
+        return self['Server']['Valid-Until'] < when
+
+    def isValidAt(self, when):
+        """Return true iff this ServerInfo is valid at time 'when'."""
+        #XXXX002 test
+        return (self['Server']['Valid-After'] <= when <=
+                self['Server']['Valid-Until'])
+
+    def isValidFrom(self, startAt, endAt):
+        """Return true iff this ServerInfo is valid at all time from 'startAt'
+           to 'endAt'."""
+        #XXXX002 test
+        return (startAt <= self['Server']['Valid-After'] and
+                self['Server']['Valid-Until'] <= endAt)
+
+    def isValidAtPartOf(self, startAt, endAt):
+        """Return true iff this ServerInfo is valid at some time between
+           'startAt' and 'endAt'."""
+        #XXXX002 test
+        va = self['Server']['Valid-After']
+        vu = self['Server']['Valid-Until']
+        return ((startAt <= va and va <= endAt) or
+                (startAt <= vu and vu <= endAt) or
+                (va <= startAt and endAt <= vu))
+
+    def isNewerThan(self, other):
+        """Return true iff this ServerInfo was published after 'other',
+           where 'other' is either a time or a ServerInfo."""
+        #XXXX002 test
+        if isinstance(other, ServerInfo):
+            other = other['Server']['Published']
+        return self['Server']['Published'] > other
+
+#----------------------------------------------------------------------
+
+#DOCDOC 
+_server_header_re = re.compile(r'^\[\s*Server\s*\]\s*\n', re.M)
+class ServerDirectory:
+    #DOCDOC
+    """Minimal client-side implementation of directory parsing.  This will
+       become very inefficient when directories get big, but we won't have
+       that problem for a while."""
+   
+    def __init__(self, string=None, fname=None):
+        if string:
+            contents = string
+        elif fname.endswith(".gz"):
+            # XXXX test!
+            f = gzip.GzipFile(fname, 'r')
+            contents = f.read()
+            f.close()
+        else:
+            f = open(fname)
+            contents = f.read()
+            f.close()
+        
+        contents = _abnormal_line_ending_re.sub("\n", contents)
+
+        # First, get the digest.  Then we can break everything up.
+        digest = _getDirectoryDigestImpl(contents)
+        
+        # This isn't a good way to do this, but what the hey.
+        sections = _server_header_re.split(contents)
+        del contents
+        headercontents = sections[0]
+        servercontents = [ "[Server]\n%s"%s for s in sections[1:] ]
+
+        self.header = _DirectoryHeader(headercontents, digest)
+        self.servers = [ ServerInfo(string=s) for s in servercontents ]
+
+    def getServers(self):
+        return self.servers
+
+    def __getitem__(self, item):
+        return self.header[item]
+
+class _DirectoryHeader(mixminion.Config._ConfigFile):
+    "DOCDOC"
+    _restrictFormat = 1
+    _syntax = {
+        'Directory': { "__SECTION__": ("REQUIRE", None, None),
+                       "Version": ("REQUIRE", None, None),
+                       "Published": ("REQUIRE", C._parseTime, None),
+                       "Valid-After": ("REQUIRE", C._parseDate, None),
+                       "Valid-Until": ("REQUIRE", C._parseDate, None),
+                       },
+        'Signature': {"__SECTION__": ("REQUIRE", None, None),
+                 "DirectoryIdentity": ("REQUIRE", C._parsePublicKey, None),
+                 "DirectoryDigest": ("REQUIRE", C._parseBase64, None),
+                 "DirectorySignature": ("REQUIRE", C._parseBase64, None),
+                      }
+        }
+
+    def __init__(self, contents, expectedDigest):
+        self.expectedDigest = expectedDigest
+        mixminion.Config._ConfigFile.__init__(self, string=contents)
+
+    def validate(self, sections, entries, lines, contents):
+        direc = sections['Directory']
+        if direc['Version'] != "0.1":
+            raise ConfigError("Unrecognized directory version")
+        if direc['Published'] > time.time() + 600:
+            raise ConfigError("Directory published in the future")
+        if direc['Valid-Until'] <= direc['Valid-After']:
+            raise ConfigError("Directory is never valid")
+
+        sig = sections['Signature']
+        identityKey = sig['DirectoryIdentity']
+        identityBytes = identityKey.get_modulus_bytes()
+        if not (MIN_IDENTITY_BYTES <= identityBytes <= MAX_IDENTITY_BYTES):
+            raise ConfigError("Invalid length on identity key")
+        
+        # Now, at last, we check the digest
+        if self.expectedDigest != sig['DirectoryDigest']:
+            raise ConfigError("Invalid digest")
+
+        try:
+            signedDigest = pk_check_signature(sig['DirectorySignature'], 
+                                              identityKey)
+        except CryptoError:
+            raise ConfigError("Invalid signature")
+        if self.expectedDigest != signedDigest:
+            raise ConfigError("Signed digest was incorrect")
 
 #----------------------------------------------------------------------
 def getServerInfoDigest(info):

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- test.py	31 Dec 2002 04:48:47 -0000	1.52
+++ test.py	3 Jan 2003 05:14:47 -0000	1.53
@@ -548,6 +548,13 @@
         # Fail if there is too much padding
         self.failUnlessRaises(_ml.CryptoError,p.crypt,x+"ZZZ",1,1)
 
+        ###
+        # Test key equality and fingerprinting.
+        self.assert_(pk_same_public_key(p, p))
+        self.assert_(not pk_same_public_key(p, getRSAKey(2,1024)))
+        self.assert_(len(pk_fingerprint(p))==40)
+        self.assertNotEquals(pk_fingerprint(p), 
+                             pk_fingerprint(getRSAKey(2,1024)))
         ####
         # Test key encoding
         padhello = _ml.add_oaep_padding("Hello", "B", 128)
@@ -560,6 +567,9 @@
             # decode(encode(x)) encrypts the same as x.
             self.assertEquals(p.crypt(padhello,public,1),
                               p2.crypt(padhello,public,1))
+            self.assert_(pk_same_public_key(p, p2))
+            self.assertEquals(pk_fingerprint(p),
+                              pk_fingerprint(p2))
 
         # encoding public keys to/from their moduli.
         self.assertEquals(p.get_modulus_bytes(),1024 >> 3)
@@ -2949,6 +2959,8 @@
         # Time
         tm = C._parseTime("2001/12/25 06:15:10")
         self.assertEquals(time.gmtime(tm)[:6], (2001,12,25,6,15,10))
+        # nicknames
+        self.assertEquals(C._parseNickname("Mrs.Premise"), "Mrs.Premise")
 
         SC = mixminion.server.ServerConfig
         # Fractions
@@ -2995,6 +3007,11 @@
         fails(C._parseDate, "2000/50/01 12:12:12")
         fails(C._parseTime, "2000/50/01 12:12:12")
         fails(C._parseTime, "2000/50/01 12:12:99")
+        fails(C._parseNickname, "Mrs Premise")
+        fails(C._parseNickname, "../../../AllYourBase")
+        fails(C._parseNickname, "Z"*129)
+        fails(C._parseNickname, ""*129)
+        fails(C._parseNickname, "; rm -f /etc/important;")
 
         nonexistcmd = '/file/that/does/not/exist'
         if not os.path.exists(nonexistcmd):
@@ -3035,7 +3052,7 @@
 EncryptPrivateKey: no
 Homedir: %s
 Mode: relay
-Nickname: The Server
+Nickname: The_Server
 Contact-Email: a@b.c
 Comments: This is a test of the emergency
    broadcast system
@@ -3066,7 +3083,6 @@
 Nickname: fred-the-bunny
 """
 
-
 class ServerInfoTests(unittest.TestCase):
     def testServerInfoGen(self):
         # Try generating a serverinfo and see if its values are as expected.
@@ -3089,7 +3105,7 @@
         eq = self.assertEquals
         eq(info['Server']['Descriptor-Version'], "0.1")
         eq(info['Server']['IP'], "192.168.0.1")
-        eq(info['Server']['Nickname'], "The Server")
+        eq(info['Server']['Nickname'], "The_Server")
         self.failUnless(0 <= time.time()-info['Server']['Published'] <= 120)
         self.failUnless(0 <= time.time()-info['Server']['Valid-After']
                           <= 24*60*60)
@@ -3150,6 +3166,8 @@
         eq(info['Server']['Digest'], loaded['Server']['Digest'])
         eq(info['Server']['Identity'].get_public_key(),
            loaded['Server']['Identity'].get_public_key())
+        eq(info['Server']['Published'], loaded['Server']['Published'])
+        eq(info.isValidated(), loaded.isValidated())
 
         # Now with a shorter configuration
         try:
@@ -3184,10 +3202,165 @@
                               mixminion.ServerInfo.ServerInfo,
                               None, badSig)
 
-# FFFF We *must* have tests for invalid server descriptors
+    def test_directory(self):
+        eq = self.assertEquals
+        examples = getExampleServerDescriptors()
+        ServerList = mixminion.directory.ServerList.ServerList
+        ServerDirectory = mixminion.ServerInfo.ServerDirectory
+        baseDir = mix_mktemp()
+        dirArchiveDir = os.path.join(baseDir, "dirArchive")
+        lst = ServerList(baseDir)
+        
+        identity = Crypto.pk_generate(2048)
+
+        now = time.time()
+        hourLater = now + 60*60
+        oneDay = 60*60*24
+        hours23 = 60*60*23
+        dayLater = now + 60*60*24
+        # Try a couple of simple inserts
+        lst.importServerInfo(examples["Fred"][1]) # from day -9 through day 0.
+        lst.importServerInfo(examples["Fred"][3]) # from day 11 through day 20
+        lst.importServerInfo(examples["Lola"][0]) # from day -2 through day 2
+        lst.importServerInfo(examples["Lola"][1]) # From day 0 through day 4.
+        # Now, check whether the guts of lst are correct.
+        eq(len(lst.servers), 4)
+        eq(len(lst.serversByNickname), 2)
+        eq(len(lst.serversByNickname['Fred']), 2)
+        eq(len(lst.serversByNickname['Lola']), 2)
+        eq(readFile(os.path.join(baseDir, "servers",
+                                 lst.serversByNickname['Fred'][0])),
+           examples["Fred"][1])
+        # Now generate a directory...
+        lst.generateDirectory(now, dayLater, 0,
+                              identity, now)
+        # (Fred1, and Lola0, and Lola1 should get included.)
+        d = readFile(lst.getDirectoryFilename())
+        self.assert_(d.startswith("[Directory]\n"))
+        eq(3, d.count("[Server]\n"))
+        self.assert_(stringContains(d, examples["Fred"][1]))
+        self.assert_(stringContains(d, examples["Lola"][0]))
+        self.assert_(stringContains(d, examples["Lola"][1]))
+
+        # Did a backup directory get made?
+        eq(1, len(os.listdir(dirArchiveDir)))
+        # Validate the directory, and check that values are as expected.
+        sd = ServerDirectory(d)
+        eq(len(sd.getServers()), 3)
+        eq(sd["Directory"]["Version"], "0.1")
+        eq(sd["Directory"]["Published"], int(now))
+        eq(sd["Directory"]["Valid-After"], previousMidnight(now))
+        eq(sd["Directory"]["Valid-Until"], previousMidnight(dayLater+1))
+        eq(sd["Signature"]["DirectoryIdentity"].get_public_key(),
+           identity.get_public_key())
+
+        # Try changing the directory, and verify that it doesn't check out
+        dBad = d.replace("Fred", "Dref")
+        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)
+        # Bad digest.
+        dBad = re.compile(r"^DirectoryDigest: ........", re.M).sub(
+            "DirectoryDigest: XXXXXXXX", d)
+        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)
+        # Bad signature.
+        dBad = re.compile(r"^DirectorySignature: ........", re.M).sub(
+            "Directory: XXXXXXXX", d)
+        self.failUnlessRaises(ConfigError, ServerDirectory, dBad)
+
+        # Can we use messed-up spaces and line-endings?
+        ServerDirectory(d.replace("\n", "\r\n"))
+        ServerDirectory(d.replace("\n", "\r"))
+        ServerDirectory(d.replace("Fred", "Fred  "))
+
+        ### Now, try rescanning the directory.
+        lst = ServerList(baseDir)
+        eq(len(lst.servers), 4)
+        eq(len(lst.serversByNickname), 2)
+        eq(len(lst.serversByNickname['Fred']), 2)
+        eq(len(lst.serversByNickname['Lola']), 2)
+        lst.generateDirectory(now, dayLater, 0,
+                              identity)
+        d2 = readFile(lst.getDirectoryFilename())
+        sd2 = ServerDirectory(d2)
+        self.assertEquals(3, len(sd2.getServers()))
+
+        # Now try cleaning servers.   First, make sure we can't insert
+        # an expired server.
+        self.failUnlessRaises(MixError, 
+                              lst.importServerInfo, examples["Fred"][0])
+        # Now, make sure we can't insert a superseded server.
+        lst.importServerInfo(examples["Bob"][3])
+        lst.importServerInfo(examples["Bob"][4])
+        self.failUnlessRaises(MixError, 
+                              lst.importServerInfo, examples["Bob"][1])
+        # Now, start with a fresh list, so we can try superceding bob later.
+        baseDir = mix_mktemp()
+        archiveDir = os.path.join(baseDir, "archive")
+        serverDir = os.path.join(baseDir, "servers")
+        lst = ServerList(baseDir)
+        # Make sure that we don't remove the last of a given server.
+        lst.importServerInfo(examples["Lisa"][1]) # Valid for 2 days.
+        lst.clean(now=now+60*60*24*100) # Very far in the future
+        eq(1, len(lst.servers))
+        eq(0, len(os.listdir(archiveDir)))
+        # But we _do_ remove expired servers if others exist.
+        lst.importServerInfo(examples["Lisa"][2]) # Valid from 5...7.
+        eq(2, len(lst.servers))
+        eq(2, len(lst.serversByNickname["Lisa"]))
+        lst.clean(now=now+60*60*24*100) # Very far in the future.
+        eq(1, len(lst.servers))
+        eq(1, len(lst.serversByNickname["Lisa"]))
+        eq(readFile(os.path.join(serverDir, lst.serversByNickname["Lisa"][0])),
+           examples["Lisa"][2])
+        eq(1, len(os.listdir(archiveDir)))
+        eq(1, len(os.listdir(serverDir)))
+        eq(readFile(os.path.join(archiveDir, os.listdir(archiveDir)[0])),
+           examples["Lisa"][1])
+
+        # (Make sure that knownOnly works: failing case.)
+        self.failUnlessRaises(MixError, lst.importServerInfo,
+                              examples["Bob"][0], 1)
+
+        ### Now test the removal of superceded servers.  
+        # Clean out archiveDir first so we can see what gets removed.
+        os.unlink(os.path.join(archiveDir, os.listdir(archiveDir)[0]))
+        # Add a bunch of unconflicting Bobs.
+        lst.importServerInfo(examples["Bob"][0]) # From -2 to 1
+        # (Make sure that knownOnly works: succeeding case.
+        lst.importServerInfo(examples["Bob"][1], 1) # From  2 to 5
+        lst.importServerInfo(examples["Bob"][2]) # From  6 to 9
+        lst.importServerInfo(examples["Bob"][3]) # Newer, from 0 to 3
+        eq(5, len(lst.servers))
+        # Right now, nothing is superceded or expired
+        lst.clean()
+        eq(5, len(os.listdir(serverDir)))
+        eq(4, len(lst.serversByNickname["Bob"]))
+        lst.importServerInfo(examples["Bob"][4]) # Newer, from 4 to 7.
+        # Now "Bob1" is superseded.
+        lst.clean()
+        eq(1, len(os.listdir(archiveDir)))
+        eq(4, len(lst.serversByNickname["Bob"]))
+        eq(5, len(os.listdir(serverDir)))
+        eq(5, len(lst.servers))
+        eq(4, len(lst.serversByNickname["Bob"]))
+        eq(readFile(os.path.join(archiveDir, os.listdir(archiveDir)[0])),
+           examples["Bob"][1])
+        for fn in lst.serversByNickname["Bob"]:
+            fn = os.path.join(serverDir, fn)
+            self.assertNotEquals(readFile(fn), examples["Bob"][1])
+        # Now try rescanning...
+        lst = ServerList(baseDir)
+        eq(5, len(lst.servers))
+        eq(4, len(lst.serversByNickname["Bob"]))
+        # ... adding a new bob...
+        lst.importServerInfo(examples["Bob"][5])
+        eq(6, len(lst.servers))
+        # ... and watching another old bob get bonked off.
+        lst.clean()
+        eq(5, len(lst.servers))
+        eq(2, len(os.listdir(archiveDir)))
 
 #----------------------------------------------------------------------
-# Modules annd ModuleManager
+# Modules and ModuleManager
 
 # Text of an example module that we load dynamically.
 EXAMPLE_MODULE_TEXT = \
@@ -3425,9 +3598,11 @@
         ####
         # Tests escapeMessageForEmail
         self.assert_(stringContains(eme(message, None), message))
-        expect = "BEGINS\n"+base64.encodestring(binmessage)+"====="
+        expect = "BEGINS ============\n"+\
+                 base64.encodestring(binmessage)+"====="
         self.assert_(stringContains(eme(binmessage, None), expect))
-        expect = "BEGINS\nDecoding handle: "+base64.encodestring(tag)+\
+        expect = "BEGINS ============\nDecoding handle: "+\
+                 base64.encodestring(tag)+\
                  base64.encodestring(encoded)+"====="
         self.assert_(stringContains(eme(encoded, tag), expect))
 
@@ -3454,12 +3629,12 @@
 This message is not in plaintext.  It's either 1) a reply; 2) a forward
 message encrypted to you; or 3) junk.
 
-============ ANONYMOUS MESSAGE BEGINS
+============ ANONYMOUS MESSAGE BEGINS ============
 Decoding handle: eHh4eHh4eHh4eHh4eHh4eHh4eHg=
 7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v
 +s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6
 zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3g==
-============ ANONYMOUS MESSAGE ENDS
+============= ANONYMOUS MESSAGE ENDS =============
 """
 
 class ModuleTests(unittest.TestCase):
@@ -3826,12 +4001,12 @@
     [ "Lola",     "5 days",  "10.0.0.7", (-2,0,5),      (MBOX_TYPE,) ],
     [ "Joe",      "20 days", "10.0.0.8", (-15,5,25),    (SMTP_TYPE,) ],
     [ "Alice",    "8 days",  "10.0.0.9", (-3,5,13),     () ],
-    [ "Bob",      "11 days", "10.0.0.10", (-10,-1,6),   () ],
+    [ "Bob",      "4 days",  "10.0.0.10", (-2, 2, 6, 'X', 0, 4, -3), () ],
     [ "Lisa",     "3 days",  "10.0.0.11", (-10,-1,5),   () ],
 ]
 
 def getExampleServerDescriptors():
-    """Helper function: generate a list of list of ServerInfo objects based
+    """Helper function: generate a map of list of ServerInfo objects based
        on the values of _EXAMPLE_DESCRIPTORS_INP"""
     if _EXAMPLE_DESCRIPTORS:
         return _EXAMPLE_DESCRIPTORS
@@ -3843,10 +4018,13 @@
     sys.stdout.flush()
 
     # For each server...
+    serveridx = 0
     for (nickname, lifetime, ip, starting, types) in _EXAMPLE_DESCRIPTORS_INP:
         # Generate a config file
         homedir = mix_mktemp()
         conf = EX_SERVER_CONF_TEMPLATE % locals()
+        identity = getRSAKey(serveridx%3,2048)
+        serveridx += 1
         for t in types:
             if t == MBOX_TYPE:
                 addrf = mix_mktemp()
@@ -3867,13 +4045,18 @@
             resumeLog()
             pass
 
-        # Now, for each starting time, generate a server desciprtor.x
+        # Now, for each starting time, generate a server desciprtor.
         _EXAMPLE_DESCRIPTORS[nickname] = []
+        publishing = now
         for n in xrange(len(starting)):
+            if starting[n] == 'X':
+                publishing += 60
+                continue
             k = "tst%d"%n
             validAt = previousMidnight(now + 24*60*60*starting[n])
-            gen(config=conf, identityKey=getRSAKey(n%3,2048), keyname=k,
-                keydir=tmpkeydir, hashdir=tmpkeydir, validAt=validAt)
+            gen(config=conf, identityKey=identity, keyname=k,
+                keydir=tmpkeydir, hashdir=tmpkeydir, validAt=validAt,
+                now=publishing)
 
             sd = os.path.join(tmpkeydir,"key_"+k,"ServerDesc")
             _EXAMPLE_DESCRIPTORS[nickname].append(readFile(sd))
@@ -3886,7 +4069,6 @@
 
 # variable to hold the latest instance of FakeBCC.
 BCC_INSTANCE = None
-
 
 class ClientMainTests(unittest.TestCase):
     def testTrivialKeystore(self):