[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[minion-cvs] Lots of documentation work and code cleanup.



Update of /home/minion/cvsroot/src/minion/lib/mixminion/directory
In directory moria.mit.edu:/tmp/cvs-serv32566/lib/mixminion/directory

Modified Files:
	DirMain.py Directory.py ServerInbox.py ServerList.py 
Log Message:
Lots of documentation work and code cleanup.

ClientMain:
- Refactor path selection so it's easier understand.

Main: 
- Add a new banner.

ServerInbox:
- Actually use ServerQueuedException as documented.

ServerMain:
- Generate keys when keygen is needed, not when key rotation is needed!
- Flush hashlogs on SIGHUP.
- Simplify USAGE code.
- Remove last vestiges of 'stop-server' and 'reload-server' homonyms.

ServerQueue:
- Put off memory-saving work until 0.0.5; it's too invasive for now.

*.py:
- Comment code, resolve XXXXs and DOCDOCs.



Index: DirMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/DirMain.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- DirMain.py	29 May 2003 05:32:11 -0000	1.15
+++ DirMain.py	6 Jun 2003 06:04:58 -0000	1.16
@@ -22,22 +22,26 @@
 USAGE = """\
 Usage: mixminion dir <command>
    Where 'command' is one of:
-      initialize [DOCDOC]
-      import-new <nickname>  [Import a descriptor for a new server]
-      list [DOCDOC]
-      update [DOCDOC]
+      initialize               [Set up a new set of directory files]
+      import-new <nickname>    [Import a descriptor for a new server]
+      list                     [List descriptors waiting to be imported]
+      update                   [Process updates for currently known servers]
       generate                 [Generate and sign a new directory]
       fingerprint              [Return the fingerprint of this directory's pk]
 """.strip()
 
 def getDirectory():
+    """Return the Directory object for this directory.  Looks for a
+       configuration file first in $MINION_DIR_CONF, then in
+       ~/.mixminion_dir.cf, then in /etc/mixminion_dir.cf.
+    """
     fn = os.environ.get('MINION_DIR_CONF')
     if not fn:
         fn = os.path.expanduser("~/.mixminion_dir.cf")
         if not os.path.exists(fn):
             fn = None
     if not fn:
-        fn = "/etc/mixion_dir.cf"
+        fn = "/etc/mixminion_dir.cf"
         if not os.path.exists(fn):
             fn = None
     if not fn:
@@ -56,6 +60,7 @@
     sys.exit(0)
 
 def cmd_init(args):
+    """[Entry point] Set up a new set of directory files."""
     if args:
         raise UIError("mixminion dir initialize takes no arguments")
     
@@ -65,6 +70,9 @@
     d.getInbox()
 
 def cmd_update(args):
+    """[Entry point] Process updates for currently known servers: copies
+       descriptors from the Inbox to ServerList.  This can be run automatically
+       as part of a cron job."""
     if args:
         raise UIError("mixminion dir update takes no arguments")
     
@@ -74,6 +82,7 @@
     inbox.acceptUpdates(serverList)
 
 def cmd_list(args):
+    """[Entry point] List descriptors waiting to be imported."""
     if args:
         raise UIError("mixminion dir list takes no arguments")
 
@@ -82,6 +91,7 @@
     inbox.listNewPendingServers(sys.stdout)
 
 def cmd_import(args):
+    """[Entry point] Import descriptors for new servers, by nickname."""
     d = getDirectory()
     inbox = d.getInbox()
     serverList = d.getServerList()
@@ -103,6 +113,8 @@
     print "\n%s servers imported, %s rejected." %(good,bad)
 
 def cmd_generate(args):
+    """[Entry point] generate a fresh directory.  Can be run from a cron
+       job."""
     if args:
         raise UIError("mixminion dir generate takes no arguments")
 
@@ -153,6 +165,8 @@
     print "Published."
 
 def cmd_fingerprint(args):
+    """[Entry point] Print the fingerprint for this directory's key."""
+
     if args:
         raise UIError("mixminion dir fingerprint takes no arguments")
     d = getDirectory()
@@ -168,6 +182,7 @@
                 }
 
 def main(cmd, args):
+    """[Entry point] Multiplex among subcommands."""
     if len(args)<1 or ('-h', '--help') in args:
         usageAndExit()
     command = args[0]

Index: Directory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/Directory.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- Directory.py	29 May 2003 05:32:11 -0000	1.9
+++ Directory.py	6 Jun 2003 06:04:58 -0000	1.10
@@ -3,8 +3,7 @@
 
 """mixminion.directory.Directory
 
-   DOCDOC
-
+   General purpose code for directory servers.
    """
 
 __all__ = [ 'ServerList', 'MismatchedID', 'DirectoryConfig', 'Directory' ]
@@ -19,7 +18,38 @@
      formatBase64, writePickled, readPickled
 
 class Directory:
+    """Wraper class for directory filestores.
+
+       Currently, a directory server keeps two filestores: an 'ServerInbox'
+       that contains servers which have been uploaded but not yet inserted
+       into the directory, and a 'ServerList' which contains the servers in
+       the directory along with the directory's private keys and so on.
+
+       A directory server also keeps an 'IDCache' that's readable by the CGI
+       user and read/writable by the directory user.  It maps nicknames to
+       identity keys.
+
+       The 'ServerInbox' is readable and (mostly) writable by the CGI user.
+       The 'ServerList' is private, and only readable by the directory user.
+       (Both of these users must have a group in common.)
+
+       This class uses a DirectoryConfig to initialize a ServerList and
+       ServerInbox as appropriate.
+
+       Layout:
+          BASEDIR/dir            [Base for ServerList.]
+          BASEDIR/inbox          [Base for ServerInbox.]
+          BASEDIR/identity_cache [File for IDCache.]
+    """
+    ##Fields:
+    # config: a DirectoryConfig instance
+    # location: the base of the directory server's files.
+    # inboxBase, directoryBase, cacheFile: filenames for the components
+    #     of this directory.
+    # cache, inbox, serverList: the actual components of this directory.
+    #     None until first initialized.
     def __init__(self, config=None, location=None):
+        """Initialize a new Directory object from a given config object."""
         self.config = config
         if config and not location:
             self.location = location = config['Directory-Store']['Homedir']
@@ -34,6 +64,7 @@
         self.serverList = None
          
     def setupDirectories(self):
+        """Create a new tree of dirs with appropriate permissions."""
         me = os.getuid()
         roledict = { 0: "root",
                      self.config.dir_uid: "dir",
@@ -52,6 +83,7 @@
         cgi_gid = self.config.cgi_gid
 
         for fn, uid, gid, mode, recurse in [
+            #Dirname              UID      #GID     mode  recurse
             (self.location,       dir_uid, cgi_gid, 0750, 1),
             (self.directoryBase,  dir_uid, dir_gid, 0700, 0),
             (ib,                  dir_uid, cgi_gid, 0750, 0),
@@ -74,32 +106,39 @@
         self._setCacheMode()
 
     def getIDCache(self):
+        """Return the IDCache for this directory."""
         if not self.cache:
             self.cache = IDCache(self.cacheFile,self._setCacheMode)
         return self.cache
 
     def _setCacheMode(self):
+        """Make sure that the IDCache is stored with the write uid, gid,
+           and permissions."""
         _set_uid_gid_mode(self.cacheFile,
                           self.config.dir_uid,
                           self.config.cgi_gid,
                           0640)
 
     def getConfig(self):
+        """Return the DirectoryConfig for this directory."""
         return self.config
 
     def getServerList(self):
+        """Return the ServerList for this directory"""
         if not self.serverList:
             from mixminion.directory.ServerList import ServerList
             self.serverList = ServerList(self.directoryBase, self.getIDCache())
         return self.serverList
 
     def getInbox(self):
+        """Return the ServerInbox for this directory"""
         if not self.inbox:
             from mixminion.directory.ServerInbox import ServerInbox
             self.inbox = ServerInbox(self.inboxBase, self.getIDCache())
         return self.inbox
 
     def getIdentity(self):
+        """Return the identity key for this directory."""
         _ = self.getServerList()
         fname = os.path.join(self.directoryBase, "identity")
         if not os.path.exists(fname):
@@ -111,6 +150,7 @@
             return mixminion.Crypto.pk_PEM_load(fname)
             
 class DirectoryConfig(C._ConfigFile):
+    """Configuration file for a directory server."""
     _restrictFormat = 0
     _restrictKeys = 1
     _syntax = {
@@ -141,6 +181,7 @@
         cgiuser = ds_sec['CGIUser'].strip().lower()
         cgigrp = ds_sec['CGIGroup'].strip().lower()
 
+        # Make sure that all the users and groups actually exist.
         try:
             dir_pwent = pwd.getpwnam(diruser)
         except KeyError:
@@ -159,6 +200,7 @@
         self.cgi_uid = cgi_pwent[2]
         self.cgi_gid = cgi_grpent[2]
 
+        # Find all members in the CGI group.
         groupMembers = cgi_grpent[3][:]
         for pwent in (dir_pwent, cgi_pwent):
             if pwent[3] == self.cgi_gid:
@@ -166,6 +208,8 @@
 
         groupMembers = [ g.lower().strip() for g in groupMembers ]
 
+        # Make sure that the directory user and the CGI user are both in
+        # the CGI group.
         if diruser not in groupMembers:
             raise C.ConfigError("User %s is not in group %s"
                                 %(diruser, cgigrp))
@@ -174,29 +218,44 @@
                                 %(cgiuser, cgigrp))
 
 class MismatchedID(Exception):
+    """Exception class: raised when the identity key on a new server
+       descriptor doesn't match the identity key known for that nickname."""
     pass
 
 class IDCache:
-    """DOCDOC"""
+    """Cache to hold a set of nickname->identitye key mappings"""
+    ##Fields:
+    # cache: map from lowercased nickname to ID fingerprint.
+    # location: filename to hold pickled cache.
+    # dirty: are all the values in 'self.cache' flushed to disk? (boolean)
+    # postSave: None, or a function to call after every save.
+    ##Pickled Format:
+    # ("V0", {lcnickname -> ID Fingerprint} )
     def __init__(self, location, postSave=None):
+        """Create an identity cache object.
+
+           location -- name of file to hold pickled cache.
+           postSave -- optionally, a function to be called after every
+              save."""
         self.location = location
         self.dirty = 0
         self.cache = None
         self.postSave = postSave
 
     def emptyCache(self):
+        """Remove all entries from this cache."""
         self.dirty = 1
         self.cache = {}
 
     def containsID(self, nickname, ID):
-        nickname = nickname.lower()
-        try:
-            return self.__containsID(nickname, ID)
-        except TypeError:
-            self.load()
-            return self.__containsID(nickname, ID)
+        """Check the identity for the server named 'nickname'.  If the
+           server is not known, return false. If the identity matches the
+           identity key fingerprint 'ID', return true.  If the server is
+           known, but the identity does not match, raise MismatchedID.
+        """
+        if self.cache is None: self.load()
 
-    def __containsID(self, lcnickname, ID):
+        lcnickname = nickname.lower()
         try:
             if self.cache[lcnickname] != ID:
                 raise MismatchedID()
@@ -205,35 +264,42 @@
             return 0
 
     def containsServer(self, server):
+        """Check the identity key contained in a server descriptor.  Return
+           true if the server is known, false if the server unknown, and
+           raise MismatchedID if the server is known but its ID is
+           incorrect."""
         nickname = server.getNickname()
         ID = getIDFingerprint(server)
         return self.containsID(nickname, ID)
 
     def insertID(self, nickname, ID):
-        nickname = nickname.lower()
-        self.dirty = 1
-        try:
-            self.__insertID(nickname, ID)
-        except AttributeError:
-            self.load()
-            self.__insertID(nickname, ID)
+        """Record the server named 'nickname' as having an identity key
+           with fingerprint 'ID'.  If the server already haves a different
+           ID, raise MismatchedID."""
+        if self.cache is None: self.load()
 
-    def __insertID(self, lcnickname, ID):
+        lcnickname = nickname.lower()
+        self.dirty = 1
         old = self.cache.get(lcnickname)
         if old and old != ID:
             raise MismatchedID()
         self.cache[lcnickname] = ID
 
     def insertServer(self, server):
+        """Record the identity key of ServerInfo 'server'.  If another
+           server with the same nickname and a different identity key is
+           already known, raise MismatchedID."""
         nickname = server.getNickname()
         ID = getIDFingerprint(server)
         self.insertID(nickname, ID)
 
     def flush(self):
+        """If any entries in the cache are new, write the cache to disk."""
         if self.dirty:
             self.save()
 
     def load(self):
+        """Re-read the cache from disk."""
         if not os.path.exists(self.location):
             LOG.info("No ID cache; will create")
             self.cache = {}
@@ -253,6 +319,9 @@
         self.cache = data
         
     def save(self):
+        """Write the cache to disk."""
+        if self.cache is None:
+            return
         writePickled(self.location,
                      ("V0", self.cache),
                      0640)
@@ -261,15 +330,20 @@
         self.dirty = 0
             
 def getIDFingerprint(server):
-    """DOCDOC"""
+    """Given a ServerInfo, return the fingerprint of its identity key.
+
+       We compute fingerprints by taking the ASN.1 encoding of the key,
+       then taking the SHA1 hash of the encoding."""
     ident = server.getIdentity()
     return mixminion.Crypto.sha1(
         mixminion.Crypto.pk_encode_public_key(ident))
 
 def _set_uid_gid_mode(fn, uid, gid, mode):
+    """Change the permissions on the file named 'fname', so that fname
+       is owned by user 'uid' and group 'gid', and has permissions 'mode'.
+    """
     st = os.stat(fn)
     if st[stat.ST_UID] != uid or st[stat.ST_GID] != gid:
         os.chown(fn, uid, gid)
     if (st[stat.ST_MODE] & 0777) != mode:
         os.chmod(fn, mode)
-    

Index: ServerInbox.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerInbox.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- ServerInbox.py	5 Jun 2003 05:24:23 -0000	1.7
+++ ServerInbox.py	6 Jun 2003 06:04:58 -0000	1.8
@@ -3,8 +3,9 @@
 
 """mixminion.directory.ServerInbox
 
-   DOCDOC
-
+   A ServerInbox holds server descriptors received from the outside world
+   that are not yet ready to be included in the directory.  It is designed
+   to be written to by an untrusted user (e.g., CGI).
    """
 
 __all__ = [ 'ServerInbox' ]
@@ -19,7 +20,17 @@
 from mixminion.directory.ServerList import _writeServer, _readServer
 
 class ServerInbox:
+    """A ServerInbox holds server descriptors received from the outside
+       world that are not yet ready to be included in the directory.
+       """
+    ## Fields:
+    # newQueue: IncomingQueue object to hold descriptors for previously
+    #      unknown servers.
+    # updateQueue:  IncomingQueue object to hold descriptors for currently
+    #      known servers.
     def __init__(self, base, idCache):
+        """Initialize a ServerInbox to store its files in 'base', and
+           check server descriptors against the IDCache 'idCache'."""
         self.newQueue = IncomingQueue(os.path.join(base, "new"),
                                       os.path.join(base, "reject"))
         self.updateQueue = IncomingQueue(os.path.join(base, "updates"),
@@ -27,10 +38,18 @@
         self.idCache = idCache
 
     def receiveServer(self, text, source):
-        """DOCDOC
+        """Process a new server descriptor and store it for later action.
+           (To be run by the CGI user.)
 
-           Returns true on OK; raises UIError on failure; raises
-           ServerQueued on wait-for-admin.
+           If the server will be automatically inserted, return true.
+           If the server will be inserted (given administrator intervention),
+              raise ServerQueuedException.
+           If there is a problem, log it, and raise UIError.
+
+           text -- a string containing a new server descriptor.
+           source -- a (human readable) string describing the source
+               of the descriptor, used in error meessages.
+        
            """
         try:
             server = ServerInfo(string=text,assumeValid=0)
@@ -53,14 +72,27 @@
             LOG.info("Received previously unknown server %s from %s",
                      nickname, source)
             self.newQueue.queueIncomingServer(text,server)
-            return
+            raise ServerQueuedException(
+                "Server queued pending manaul checking")
         else:
             LOG.info("Received update for server %s from %s",
                      nickname, source)
             self.updateQueue.queueIncomingServer(text,server)
-            return
+            return 1 
 
     def _doAccept(self, serverList, q, incoming, reject, knownOnly):
+        """Helper function: move servers from an IncomingQueue into
+           a ServerList.  (To be run by the directory user.)
+
+           serverList -- an instance of ServerList
+           q -- an instance of IncomingQueue
+           incoming -- a list of [filename, serverinfo, descriptor text,
+                fingerprint] for servers to insert.
+           reject -- a list of [filename, serverinfo, desc text, fprint]
+                for servers to reject.
+           knownOnly -- boolean: accept only servers with previously
+                known identity keys?
+        """
         accepted = []
         for fname, server, text, fp in incoming: 
             try:
@@ -79,11 +111,19 @@
         q.delPendingServers(fnames)
 
     def acceptUpdates(self, serverList):
+        """Move updates for existing servers into the directory.  (To
+           be run by the directory user.)"""
         incoming = self.updateQueue.readPendingServers()
         self._doAccept(serverList, self.updateQueue, incoming, [],
                        knownOnly=1)
 
     def acceptNewServer(self, serverList, nickname):
+        """Move the descriptors for a new server with a given nickname
+           into the directory.  (To be run by a the directory user.)
+
+           If the nickname is of the format name:FINGERPRINT, then
+           only insert servers with the nickname/fingerprint pair.
+        """
         if ':' in nickname:
             nickname, fingerprint = nickname.split(":")
         else:
@@ -126,7 +166,8 @@
             serverList._unlock()
 
     def listNewPendingServers(self, f):
-        """DOCDOC"""
+        """Print a list of new servers waiting admin attention to the file
+           f."""
         incoming = self.newQueue.readPendingServers()
         # lcnickname->fp->servers
         servers = {}
@@ -155,9 +196,11 @@
                 print >>f, (format%(nickname,fp,len(s)))
 
 class IncomingQueue:
-    """DOCDOC"""
+    """Implementation helper: holds incoming server descriptors as
+       separate files in a directory."""
     def __init__(self, incomingDir, rejectDir):
-        """DOCDOC"""
+        """Create an IncomingQueue to hold incoming servers in incomingDir
+           and rejected servers in rejectDir."""
         self.incomingDir = incomingDir
         self.rejectDir = rejectDir
         if not os.path.exists(incomingDir):
@@ -166,19 +209,34 @@
             raise MixFatalError("Reject directory doesn't exist")
 
     def queueIncomingServer(self, contents, server):
-        """DOCDOC"""
+        """Write a server into the incoming directory.
+
+           contents -- the text of the server descriptor.
+           server -- the parsed server descriptor.
+        """
         nickname = server.getNickname()
         _writeServer(self.incomingDir, contents, nickname, 0644)
 
     def queueRejectedServer(self, contents, server):
+        """Write a server into the rejected directory.
+
+           contents -- the text of the server descriptor.
+           server -- the parsed server descriptor.
+        """
         nickname = server.getNickname()
         _writeServer(self.rejectDir, contents, nickname, 0644)
 
     def newServersPending(self, newServ):
-        """DOCDOC"""
+        """Return true iff there is a new server waiting in the incoming
+           directory."""
         return len(os.listdir(self.incomingDir)) > 0
 
     def readPendingServers(self):
+        """Scan all of the servers waiting in the incoming directory.  If
+           any are bad, remove them.  Return a list of
+              (filename, ServerInfo, server descriptor, ID Fingerprint)
+           tuples for all the servers in the directory.
+           """
         res = []
         for fname in os.listdir(self.incomingDir):
             path = os.path.join(self.incomingDir,fname)
@@ -186,18 +244,22 @@
                 text, server = _readServer(path)
             except MixError, e:
                 os.unlink(path)
-                LOG.warn("Removed a bad server descriptor %s from incoming dir: %s",
-                         fname, e)
+                LOG.warn(
+                    "Removed a bad server descriptor %s from incoming dir: %s",
+                    fname, e)
                 continue
             fp = formatBase64(getIDFingerprint(server))
             res.append((fname, server, text, fp))
         return res
 
     def delPendingServers(self, fnames):
+        """Remove a list of pending servers with filename 'filename' from
+           the incoming directory."""
         for fname in fnames:
             if not tryUnlink(os.path.join(self.incomingDir, fname)):
                 LOG.warn("delPendingServers: no such server %s"%fname)
 
 class ServerQueuedException(Exception):
-    """DOCDOC"""
+    """Exception: raised when an incoming server is received for a previously
+       unknown nickname."""
     pass

Index: ServerList.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerList.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- ServerList.py	5 Jun 2003 05:24:23 -0000	1.29
+++ ServerList.py	6 Jun 2003 06:04:58 -0000	1.30
@@ -3,7 +3,8 @@
 
 """mixminion.directory.ServerList
 
-   Implements a store of serverinfos for a directory.
+   Implements a store of serverinfos for a directory, as well as functions
+   to generate and sign directories.
 
    FFFF Right now, this is about maximally slow.  There are a lot of tricks
    FFFF we could do to speed it up: not revalidating servers in our cache;
@@ -66,6 +67,7 @@
     #  servers: Map from filename within <serverDir> to ServerInfo objects.
     #  serversByNickname: A map from lowercased server nickname to
     #       lists of filenames within <serverDir>
+    #  idCache: an instance of Directory.IDCache
     ##Layout:
     #  basedir
     #     server-ids/
@@ -86,8 +88,7 @@
     #          dir-dategenerated.N ...
     #     identity
     #     .lock
-    #
-    # idCache: DOCDOC
+    
     def __init__(self, baseDir, idCache=None):
         """Initialize a ServerList to store servers under baseDir/servers,
            creating directories as needed.
@@ -130,7 +131,8 @@
             self._unlock()
 
     def learnServerID(self, server):
-        """DOCDOC"""
+        """Mark the ID for a server descriptor as the canonical
+           identity key associated with that server's nickname."""
         try:
             self._lock()
             ident = server.getIdentity()
@@ -158,8 +160,8 @@
                file containing the descriptor (possibly gzip'd)
            knownOnly -- if true, raise MixError is we don't already have
                a descriptor with this nickname.
-
-           DOCDOC
+           server -- If provided, a parsed ServerInfo corresponding to
+               'contents'.
         """
         # Raises ConfigError, MixError,
 
@@ -283,7 +285,6 @@
             # FFFF We should probably not do all of this in RAM, but
             # FFFF what the hey.  It will only matter if we have many, many
             # FFFF servers in the system.
-
             contents = [ ]
             for _, _, fn in included:
                 txt = readFile(os.path.join(self.serverDir, fn))
@@ -462,8 +463,11 @@
         self.lockfile.release()
         self.rlock.release()
 
-
 def _writeServer(directory, contents, nickname, mode=0600):
+    """Write a server descriptor in 'contents' into a new file in
+       'directory'.  The file will have permissions 'mode', and a name
+       of the form nickname-YYYYMMDDHHMMSS.n
+    """
     newFile = nickname+"-"+formatFnameTime()
     f, newFile = openUnique(os.path.join(directory, newFile), 'w', mode)
     newFile = os.path.split(newFile)[1]
@@ -472,6 +476,9 @@
     return newFile
 
 def _readServer(contents):
+    """Read a ServerInfo from the string 'contents', which is either a
+       server descriptor or the name of a file holding a descriptor.
+       Raise MixError on failure."""
     if stringContains(contents, "[Server]"):
         pass
     else: