[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Lots of documentation work and code cleanup.
Update of /home/minion/cvsroot/src/minion/lib/mixminion/directory
In directory moria.mit.edu:/tmp/cvs-serv32566/lib/mixminion/directory
Modified Files:
DirMain.py Directory.py ServerInbox.py ServerList.py
Log Message:
Lots of documentation work and code cleanup.
ClientMain:
- Refactor path selection so it's easier understand.
Main:
- Add a new banner.
ServerInbox:
- Actually use ServerQueuedException as documented.
ServerMain:
- Generate keys when keygen is needed, not when key rotation is needed!
- Flush hashlogs on SIGHUP.
- Simplify USAGE code.
- Remove last vestiges of 'stop-server' and 'reload-server' homonyms.
ServerQueue:
- Put off memory-saving work until 0.0.5; it's too invasive for now.
*.py:
- Comment code, resolve XXXXs and DOCDOCs.
Index: DirMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/DirMain.py,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- DirMain.py 29 May 2003 05:32:11 -0000 1.15
+++ DirMain.py 6 Jun 2003 06:04:58 -0000 1.16
@@ -22,22 +22,26 @@
USAGE = """\
Usage: mixminion dir <command>
Where 'command' is one of:
- initialize [DOCDOC]
- import-new <nickname> [Import a descriptor for a new server]
- list [DOCDOC]
- update [DOCDOC]
+ initialize [Set up a new set of directory files]
+ import-new <nickname> [Import a descriptor for a new server]
+ list [List descriptors waiting to be imported]
+ update [Process updates for currently known servers]
generate [Generate and sign a new directory]
fingerprint [Return the fingerprint of this directory's pk]
""".strip()
def getDirectory():
+ """Return the Directory object for this directory. Looks for a
+ configuration file first in $MINION_DIR_CONF, then in
+ ~/.mixminion_dir.cf, then in /etc/mixminion_dir.cf.
+ """
fn = os.environ.get('MINION_DIR_CONF')
if not fn:
fn = os.path.expanduser("~/.mixminion_dir.cf")
if not os.path.exists(fn):
fn = None
if not fn:
- fn = "/etc/mixion_dir.cf"
+ fn = "/etc/mixminion_dir.cf"
if not os.path.exists(fn):
fn = None
if not fn:
@@ -56,6 +60,7 @@
sys.exit(0)
def cmd_init(args):
+ """[Entry point] Set up a new set of directory files."""
if args:
raise UIError("mixminion dir initialize takes no arguments")
@@ -65,6 +70,9 @@
d.getInbox()
def cmd_update(args):
+ """[Entry point] Process updates for currently known servers: copies
+ descriptors from the Inbox to ServerList. This can be run automatically
+ as part of a cron job."""
if args:
raise UIError("mixminion dir update takes no arguments")
@@ -74,6 +82,7 @@
inbox.acceptUpdates(serverList)
def cmd_list(args):
+ """[Entry point] List descriptors waiting to be imported."""
if args:
raise UIError("mixminion dir list takes no arguments")
@@ -82,6 +91,7 @@
inbox.listNewPendingServers(sys.stdout)
def cmd_import(args):
+ """[Entry point] Import descriptors for new servers, by nickname."""
d = getDirectory()
inbox = d.getInbox()
serverList = d.getServerList()
@@ -103,6 +113,8 @@
print "\n%s servers imported, %s rejected." %(good,bad)
def cmd_generate(args):
+ """[Entry point] generate a fresh directory. Can be run from a cron
+ job."""
if args:
raise UIError("mixminion dir generate takes no arguments")
@@ -153,6 +165,8 @@
print "Published."
def cmd_fingerprint(args):
+ """[Entry point] Print the fingerprint for this directory's key."""
+
if args:
raise UIError("mixminion dir fingerprint takes no arguments")
d = getDirectory()
@@ -168,6 +182,7 @@
}
def main(cmd, args):
+ """[Entry point] Multiplex among subcommands."""
if len(args)<1 or ('-h', '--help') in args:
usageAndExit()
command = args[0]
Index: Directory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/Directory.py,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- Directory.py 29 May 2003 05:32:11 -0000 1.9
+++ Directory.py 6 Jun 2003 06:04:58 -0000 1.10
@@ -3,8 +3,7 @@
"""mixminion.directory.Directory
- DOCDOC
-
+ General purpose code for directory servers.
"""
__all__ = [ 'ServerList', 'MismatchedID', 'DirectoryConfig', 'Directory' ]
@@ -19,7 +18,38 @@
formatBase64, writePickled, readPickled
class Directory:
+ """Wraper class for directory filestores.
+
+ Currently, a directory server keeps two filestores: an 'ServerInbox'
+ that contains servers which have been uploaded but not yet inserted
+ into the directory, and a 'ServerList' which contains the servers in
+ the directory along with the directory's private keys and so on.
+
+ A directory server also keeps an 'IDCache' that's readable by the CGI
+ user and read/writable by the directory user. It maps nicknames to
+ identity keys.
+
+ The 'ServerInbox' is readable and (mostly) writable by the CGI user.
+ The 'ServerList' is private, and only readable by the directory user.
+ (Both of these users must have a group in common.)
+
+ This class uses a DirectoryConfig to initialize a ServerList and
+ ServerInbox as appropriate.
+
+ Layout:
+ BASEDIR/dir [Base for ServerList.]
+ BASEDIR/inbox [Base for ServerInbox.]
+ BASEDIR/identity_cache [File for IDCache.]
+ """
+ ##Fields:
+ # config: a DirectoryConfig instance
+ # location: the base of the directory server's files.
+ # inboxBase, directoryBase, cacheFile: filenames for the components
+ # of this directory.
+ # cache, inbox, serverList: the actual components of this directory.
+ # None until first initialized.
def __init__(self, config=None, location=None):
+ """Initialize a new Directory object from a given config object."""
self.config = config
if config and not location:
self.location = location = config['Directory-Store']['Homedir']
@@ -34,6 +64,7 @@
self.serverList = None
def setupDirectories(self):
+ """Create a new tree of dirs with appropriate permissions."""
me = os.getuid()
roledict = { 0: "root",
self.config.dir_uid: "dir",
@@ -52,6 +83,7 @@
cgi_gid = self.config.cgi_gid
for fn, uid, gid, mode, recurse in [
+ #Dirname UID #GID mode recurse
(self.location, dir_uid, cgi_gid, 0750, 1),
(self.directoryBase, dir_uid, dir_gid, 0700, 0),
(ib, dir_uid, cgi_gid, 0750, 0),
@@ -74,32 +106,39 @@
self._setCacheMode()
def getIDCache(self):
+ """Return the IDCache for this directory."""
if not self.cache:
self.cache = IDCache(self.cacheFile,self._setCacheMode)
return self.cache
def _setCacheMode(self):
+ """Make sure that the IDCache is stored with the write uid, gid,
+ and permissions."""
_set_uid_gid_mode(self.cacheFile,
self.config.dir_uid,
self.config.cgi_gid,
0640)
def getConfig(self):
+ """Return the DirectoryConfig for this directory."""
return self.config
def getServerList(self):
+ """Return the ServerList for this directory"""
if not self.serverList:
from mixminion.directory.ServerList import ServerList
self.serverList = ServerList(self.directoryBase, self.getIDCache())
return self.serverList
def getInbox(self):
+ """Return the ServerInbox for this directory"""
if not self.inbox:
from mixminion.directory.ServerInbox import ServerInbox
self.inbox = ServerInbox(self.inboxBase, self.getIDCache())
return self.inbox
def getIdentity(self):
+ """Return the identity key for this directory."""
_ = self.getServerList()
fname = os.path.join(self.directoryBase, "identity")
if not os.path.exists(fname):
@@ -111,6 +150,7 @@
return mixminion.Crypto.pk_PEM_load(fname)
class DirectoryConfig(C._ConfigFile):
+ """Configuration file for a directory server."""
_restrictFormat = 0
_restrictKeys = 1
_syntax = {
@@ -141,6 +181,7 @@
cgiuser = ds_sec['CGIUser'].strip().lower()
cgigrp = ds_sec['CGIGroup'].strip().lower()
+ # Make sure that all the users and groups actually exist.
try:
dir_pwent = pwd.getpwnam(diruser)
except KeyError:
@@ -159,6 +200,7 @@
self.cgi_uid = cgi_pwent[2]
self.cgi_gid = cgi_grpent[2]
+ # Find all members in the CGI group.
groupMembers = cgi_grpent[3][:]
for pwent in (dir_pwent, cgi_pwent):
if pwent[3] == self.cgi_gid:
@@ -166,6 +208,8 @@
groupMembers = [ g.lower().strip() for g in groupMembers ]
+ # Make sure that the directory user and the CGI user are both in
+ # the CGI group.
if diruser not in groupMembers:
raise C.ConfigError("User %s is not in group %s"
%(diruser, cgigrp))
@@ -174,29 +218,44 @@
%(cgiuser, cgigrp))
class MismatchedID(Exception):
+ """Exception class: raised when the identity key on a new server
+ descriptor doesn't match the identity key known for that nickname."""
pass
class IDCache:
- """DOCDOC"""
+ """Cache to hold a set of nickname->identitye key mappings"""
+ ##Fields:
+ # cache: map from lowercased nickname to ID fingerprint.
+ # location: filename to hold pickled cache.
+ # dirty: are all the values in 'self.cache' flushed to disk? (boolean)
+ # postSave: None, or a function to call after every save.
+ ##Pickled Format:
+ # ("V0", {lcnickname -> ID Fingerprint} )
def __init__(self, location, postSave=None):
+ """Create an identity cache object.
+
+ location -- name of file to hold pickled cache.
+ postSave -- optionally, a function to be called after every
+ save."""
self.location = location
self.dirty = 0
self.cache = None
self.postSave = postSave
def emptyCache(self):
+ """Remove all entries from this cache."""
self.dirty = 1
self.cache = {}
def containsID(self, nickname, ID):
- nickname = nickname.lower()
- try:
- return self.__containsID(nickname, ID)
- except TypeError:
- self.load()
- return self.__containsID(nickname, ID)
+ """Check the identity for the server named 'nickname'. If the
+ server is not known, return false. If the identity matches the
+ identity key fingerprint 'ID', return true. If the server is
+ known, but the identity does not match, raise MismatchedID.
+ """
+ if self.cache is None: self.load()
- def __containsID(self, lcnickname, ID):
+ lcnickname = nickname.lower()
try:
if self.cache[lcnickname] != ID:
raise MismatchedID()
@@ -205,35 +264,42 @@
return 0
def containsServer(self, server):
+ """Check the identity key contained in a server descriptor. Return
+ true if the server is known, false if the server unknown, and
+ raise MismatchedID if the server is known but its ID is
+ incorrect."""
nickname = server.getNickname()
ID = getIDFingerprint(server)
return self.containsID(nickname, ID)
def insertID(self, nickname, ID):
- nickname = nickname.lower()
- self.dirty = 1
- try:
- self.__insertID(nickname, ID)
- except AttributeError:
- self.load()
- self.__insertID(nickname, ID)
+ """Record the server named 'nickname' as having an identity key
+ with fingerprint 'ID'. If the server already haves a different
+ ID, raise MismatchedID."""
+ if self.cache is None: self.load()
- def __insertID(self, lcnickname, ID):
+ lcnickname = nickname.lower()
+ self.dirty = 1
old = self.cache.get(lcnickname)
if old and old != ID:
raise MismatchedID()
self.cache[lcnickname] = ID
def insertServer(self, server):
+ """Record the identity key of ServerInfo 'server'. If another
+ server with the same nickname and a different identity key is
+ already known, raise MismatchedID."""
nickname = server.getNickname()
ID = getIDFingerprint(server)
self.insertID(nickname, ID)
def flush(self):
+ """If any entries in the cache are new, write the cache to disk."""
if self.dirty:
self.save()
def load(self):
+ """Re-read the cache from disk."""
if not os.path.exists(self.location):
LOG.info("No ID cache; will create")
self.cache = {}
@@ -253,6 +319,9 @@
self.cache = data
def save(self):
+ """Write the cache to disk."""
+ if self.cache is None:
+ return
writePickled(self.location,
("V0", self.cache),
0640)
@@ -261,15 +330,20 @@
self.dirty = 0
def getIDFingerprint(server):
- """DOCDOC"""
+ """Given a ServerInfo, return the fingerprint of its identity key.
+
+ We compute fingerprints by taking the ASN.1 encoding of the key,
+ then taking the SHA1 hash of the encoding."""
ident = server.getIdentity()
return mixminion.Crypto.sha1(
mixminion.Crypto.pk_encode_public_key(ident))
def _set_uid_gid_mode(fn, uid, gid, mode):
+ """Change the permissions on the file named 'fname', so that fname
+ is owned by user 'uid' and group 'gid', and has permissions 'mode'.
+ """
st = os.stat(fn)
if st[stat.ST_UID] != uid or st[stat.ST_GID] != gid:
os.chown(fn, uid, gid)
if (st[stat.ST_MODE] & 0777) != mode:
os.chmod(fn, mode)
-
Index: ServerInbox.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerInbox.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- ServerInbox.py 5 Jun 2003 05:24:23 -0000 1.7
+++ ServerInbox.py 6 Jun 2003 06:04:58 -0000 1.8
@@ -3,8 +3,9 @@
"""mixminion.directory.ServerInbox
- DOCDOC
-
+ A ServerInbox holds server descriptors received from the outside world
+ that are not yet ready to be included in the directory. It is designed
+ to be written to by an untrusted user (e.g., CGI).
"""
__all__ = [ 'ServerInbox' ]
@@ -19,7 +20,17 @@
from mixminion.directory.ServerList import _writeServer, _readServer
class ServerInbox:
+ """A ServerInbox holds server descriptors received from the outside
+ world that are not yet ready to be included in the directory.
+ """
+ ## Fields:
+ # newQueue: IncomingQueue object to hold descriptors for previously
+ # unknown servers.
+ # updateQueue: IncomingQueue object to hold descriptors for currently
+ # known servers.
def __init__(self, base, idCache):
+ """Initialize a ServerInbox to store its files in 'base', and
+ check server descriptors against the IDCache 'idCache'."""
self.newQueue = IncomingQueue(os.path.join(base, "new"),
os.path.join(base, "reject"))
self.updateQueue = IncomingQueue(os.path.join(base, "updates"),
@@ -27,10 +38,18 @@
self.idCache = idCache
def receiveServer(self, text, source):
- """DOCDOC
+ """Process a new server descriptor and store it for later action.
+ (To be run by the CGI user.)
- Returns true on OK; raises UIError on failure; raises
- ServerQueued on wait-for-admin.
+ If the server will be automatically inserted, return true.
+ If the server will be inserted (given administrator intervention),
+ raise ServerQueuedException.
+ If there is a problem, log it, and raise UIError.
+
+ text -- a string containing a new server descriptor.
+ source -- a (human readable) string describing the source
+ of the descriptor, used in error meessages.
+
"""
try:
server = ServerInfo(string=text,assumeValid=0)
@@ -53,14 +72,27 @@
LOG.info("Received previously unknown server %s from %s",
nickname, source)
self.newQueue.queueIncomingServer(text,server)
- return
+ raise ServerQueuedException(
+ "Server queued pending manaul checking")
else:
LOG.info("Received update for server %s from %s",
nickname, source)
self.updateQueue.queueIncomingServer(text,server)
- return
+ return 1
def _doAccept(self, serverList, q, incoming, reject, knownOnly):
+ """Helper function: move servers from an IncomingQueue into
+ a ServerList. (To be run by the directory user.)
+
+ serverList -- an instance of ServerList
+ q -- an instance of IncomingQueue
+ incoming -- a list of [filename, serverinfo, descriptor text,
+ fingerprint] for servers to insert.
+ reject -- a list of [filename, serverinfo, desc text, fprint]
+ for servers to reject.
+ knownOnly -- boolean: accept only servers with previously
+ known identity keys?
+ """
accepted = []
for fname, server, text, fp in incoming:
try:
@@ -79,11 +111,19 @@
q.delPendingServers(fnames)
def acceptUpdates(self, serverList):
+ """Move updates for existing servers into the directory. (To
+ be run by the directory user.)"""
incoming = self.updateQueue.readPendingServers()
self._doAccept(serverList, self.updateQueue, incoming, [],
knownOnly=1)
def acceptNewServer(self, serverList, nickname):
+ """Move the descriptors for a new server with a given nickname
+ into the directory. (To be run by a the directory user.)
+
+ If the nickname is of the format name:FINGERPRINT, then
+ only insert servers with the nickname/fingerprint pair.
+ """
if ':' in nickname:
nickname, fingerprint = nickname.split(":")
else:
@@ -126,7 +166,8 @@
serverList._unlock()
def listNewPendingServers(self, f):
- """DOCDOC"""
+ """Print a list of new servers waiting admin attention to the file
+ f."""
incoming = self.newQueue.readPendingServers()
# lcnickname->fp->servers
servers = {}
@@ -155,9 +196,11 @@
print >>f, (format%(nickname,fp,len(s)))
class IncomingQueue:
- """DOCDOC"""
+ """Implementation helper: holds incoming server descriptors as
+ separate files in a directory."""
def __init__(self, incomingDir, rejectDir):
- """DOCDOC"""
+ """Create an IncomingQueue to hold incoming servers in incomingDir
+ and rejected servers in rejectDir."""
self.incomingDir = incomingDir
self.rejectDir = rejectDir
if not os.path.exists(incomingDir):
@@ -166,19 +209,34 @@
raise MixFatalError("Reject directory doesn't exist")
def queueIncomingServer(self, contents, server):
- """DOCDOC"""
+ """Write a server into the incoming directory.
+
+ contents -- the text of the server descriptor.
+ server -- the parsed server descriptor.
+ """
nickname = server.getNickname()
_writeServer(self.incomingDir, contents, nickname, 0644)
def queueRejectedServer(self, contents, server):
+ """Write a server into the rejected directory.
+
+ contents -- the text of the server descriptor.
+ server -- the parsed server descriptor.
+ """
nickname = server.getNickname()
_writeServer(self.rejectDir, contents, nickname, 0644)
def newServersPending(self, newServ):
- """DOCDOC"""
+ """Return true iff there is a new server waiting in the incoming
+ directory."""
return len(os.listdir(self.incomingDir)) > 0
def readPendingServers(self):
+ """Scan all of the servers waiting in the incoming directory. If
+ any are bad, remove them. Return a list of
+ (filename, ServerInfo, server descriptor, ID Fingerprint)
+ tuples for all the servers in the directory.
+ """
res = []
for fname in os.listdir(self.incomingDir):
path = os.path.join(self.incomingDir,fname)
@@ -186,18 +244,22 @@
text, server = _readServer(path)
except MixError, e:
os.unlink(path)
- LOG.warn("Removed a bad server descriptor %s from incoming dir: %s",
- fname, e)
+ LOG.warn(
+ "Removed a bad server descriptor %s from incoming dir: %s",
+ fname, e)
continue
fp = formatBase64(getIDFingerprint(server))
res.append((fname, server, text, fp))
return res
def delPendingServers(self, fnames):
+ """Remove a list of pending servers with filename 'filename' from
+ the incoming directory."""
for fname in fnames:
if not tryUnlink(os.path.join(self.incomingDir, fname)):
LOG.warn("delPendingServers: no such server %s"%fname)
class ServerQueuedException(Exception):
- """DOCDOC"""
+ """Exception: raised when an incoming server is received for a previously
+ unknown nickname."""
pass
Index: ServerList.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerList.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- ServerList.py 5 Jun 2003 05:24:23 -0000 1.29
+++ ServerList.py 6 Jun 2003 06:04:58 -0000 1.30
@@ -3,7 +3,8 @@
"""mixminion.directory.ServerList
- Implements a store of serverinfos for a directory.
+ Implements a store of serverinfos for a directory, as well as functions
+ to generate and sign directories.
FFFF Right now, this is about maximally slow. There are a lot of tricks
FFFF we could do to speed it up: not revalidating servers in our cache;
@@ -66,6 +67,7 @@
# servers: Map from filename within <serverDir> to ServerInfo objects.
# serversByNickname: A map from lowercased server nickname to
# lists of filenames within <serverDir>
+ # idCache: an instance of Directory.IDCache
##Layout:
# basedir
# server-ids/
@@ -86,8 +88,7 @@
# dir-dategenerated.N ...
# identity
# .lock
- #
- # idCache: DOCDOC
+
def __init__(self, baseDir, idCache=None):
"""Initialize a ServerList to store servers under baseDir/servers,
creating directories as needed.
@@ -130,7 +131,8 @@
self._unlock()
def learnServerID(self, server):
- """DOCDOC"""
+ """Mark the ID for a server descriptor as the canonical
+ identity key associated with that server's nickname."""
try:
self._lock()
ident = server.getIdentity()
@@ -158,8 +160,8 @@
file containing the descriptor (possibly gzip'd)
knownOnly -- if true, raise MixError is we don't already have
a descriptor with this nickname.
-
- DOCDOC
+ server -- If provided, a parsed ServerInfo corresponding to
+ 'contents'.
"""
# Raises ConfigError, MixError,
@@ -283,7 +285,6 @@
# FFFF We should probably not do all of this in RAM, but
# FFFF what the hey. It will only matter if we have many, many
# FFFF servers in the system.
-
contents = [ ]
for _, _, fn in included:
txt = readFile(os.path.join(self.serverDir, fn))
@@ -462,8 +463,11 @@
self.lockfile.release()
self.rlock.release()
-
def _writeServer(directory, contents, nickname, mode=0600):
+ """Write a server descriptor in 'contents' into a new file in
+ 'directory'. The file will have permissions 'mode', and a name
+ of the form nickname-YYYYMMDDHHMMSS.n
+ """
newFile = nickname+"-"+formatFnameTime()
f, newFile = openUnique(os.path.join(directory, newFile), 'w', mode)
newFile = os.path.split(newFile)[1]
@@ -472,6 +476,9 @@
return newFile
def _readServer(contents):
+ """Read a ServerInfo from the string 'contents', which is either a
+ server descriptor or the name of a file holding a descriptor.
+ Raise MixError on failure."""
if stringContains(contents, "[Server]"):
pass
else: