[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Directories implemented for client and server, but not ...
Update of /home/minion/cvsroot/src/minion/lib/mixminion/directory
In directory moria.mit.edu:/tmp/cvs-serv11194/minion/lib/mixminion/directory
Modified Files:
ServerList.py
Added Files:
DirMain.py
Log Message:
Directories implemented for client and server, but not doc'd or tested.
TODO:
- Reflect state of directory work.
setup.py:
- Run python as python -O from the default-installed script.
ClientMain.py:
- Reimplement keystore to know about directories and select paths. The new
one also caches parsed values to run a bit faster.
- Make the path interface a little more complicated.
- Simplify 'main' a bit by refactoring out the configfile and usage logic.
Config, ServerInfo:
- Builtin gzip support
Config, ServerInfo, server/ServerConfig:
- Restrict nicknames to reasonable characters
Crypto:
- Add 'fingerprint' function
- Document more.
- Add rng.pick(lst) as a shortcut for lst[rng.getInt(len(lst))]
Main, directory/DirMain:
- Add CLI for directory generation:
ServerInfo:
- Add numerous helper functions to ServerInfo
- Add ServerDirectory class to parse server directories.
test:
- Add tests for nickname validation
- Tests for pk_same_public_key and pk_fingerprint
- Tests for directory generation and parsing.
directory/ServerList:
- Debug, document, refactor
server/Modules:
- Use balanced 'banners' around email-quoted messages (Suggested by Lucky)
server/ServerKeys:
- Add 'now' argument to generateServerDescriptorAndKeys for testing
--- NEW FILE: DirMain.py ---
# Copyright 2002 Nick Mathewson. See LICENSE for licensing information.
# $Id: DirMain.py,v 1.1 2003/01/03 05:14:47 nickm Exp $
"""mixminion.directory.DirMain
CLI for mixminion directory generation.
"""
__all__ = [ ]
import gzip
import os
import shutil
import stat
import sys
import time
from mixminion.Common import createPrivateDir, formatTime, LOG
from mixminion.Crypto import init_crypto, pk_fingerprint, pk_generate, \
pk_PEM_load, pk_PEM_save
from mixminion.directory.ServerList import ServerList
USAGE = """%s -d <directory> command
Where 'command' is one of:
import <serverinfo>
import-new <serverinfo>
generate
export <filename>
remove <nickname>
fingerprint"""
def getIdentity(baseDir):
"DOCDOC"
createPrivateDir(baseDir)
fname = os.path.join(baseDir, "identity")
if not os.path.exists(fname):
print "No public key found; generating new key..."
key = pk_generate(2048)
pk_PEM_save(key, fname)
return key
else:
return pk_PEM_load(fname)
def usageAndExit(cmd):
"DOCDOC"
print >>sys.stderr, USAGE%cmd
raise "N"
sys.exit(1)
def cmd_import(cmd, base, rest):
if len(rest) != 1: usageAndExit(cmd)
lst = ServerList(base)
lst.importServerInfo(rest[0], knownOnly=1)
print >>sys.stderr, "Imported."
def cmd_import_new(cmd, base, rest):
if len(rest) != 1: usageAndExit(cmd)
lst = ServerList(base)
lst.importServerInfo(rest[0], knownOnly=0)
print >>sys.stderr, "Imported."
def cmd_generate(cmd, base, rest):
if len(rest) != 0: usageAndExit(cmd)
lst = ServerList(base)
key = getIdentity(base)
# XXXX Until we have support for automatic directory generation, we
# XXXX set the validity time to be pretty long: 2 months.
now = time.time()
twoMonthsLater = now + 60*60*24*30*2
lst.generateDirectory(startAt=now, endAt=twoMonthsLater, extraTime=0,
identityKey=key)
print >>sys.stderr, "Directory generated."
def cmd_export(cmd, base, rest):
"DOCDOC"
if len(rest) != 1: usageAndExit(cmd)
lst = ServerList(base)
fname = lst.getDirectoryFilename()
if not os.path.exists(fname):
print >>sys.stderr, "No directory has been generated"
st = os.stat(fname)
print >>sys.stderr, "Exporting directory from %s"%(
formatTime(st[stat.ST_MTIME]))
if rest[0] == '-':
f = open(fname)
d = f.read()
f.close()
sys.stdout.write(d)
elif rest[0].endswith(".gz"):
fIn = open(fname)
fOut = gzip.GzipFile(rest[0], 'w')
fOut.write(fIn.read())
fIn.close()
fOut.close()
else:
shutil.copy(fname, rest[0])
print >>sys.stderr, "Exported."
def cmd_remove(cmd, base, rest):
if len(rest) != 1: usageAndExit(cmd)
lst = ServerList(base)
lst.expungeServersByNickname(rest[0])
def cmd_fingerprint(cmd, base, rest):
if len(rest) != 0: usageAndExit(cmd)
key = getIdentity(base)
print pk_fingerprint(key)
SUBCOMMANDS = { 'import' : cmd_import,
'import-new' : cmd_import_new,
'generate' : cmd_generate,
'export' : cmd_export,
'remove' : cmd_remove,
'fingerprint' : cmd_fingerprint }
def main(cmd, args):
if len(args) < 3 or args[0] != "-d":
usageAndExit(cmd)
baseDir = args[1]
command = args[2]
if not SUBCOMMANDS.has_key(command):
print >>sys.stderr, "Unknown command", command
usageAndExit(cmd)
init_crypto()
LOG.setMinSeverity("INFO")
SUBCOMMANDS[command](cmd, baseDir, args[3:])
Index: ServerList.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerList.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- ServerList.py 31 Dec 2002 04:33:25 -0000 1.1
+++ ServerList.py 3 Jan 2003 05:14:47 -0000 1.2
@@ -20,9 +20,11 @@
from mixminion.Crypto import pk_encode_public_key, pk_same_public_key
from mixminion.Common import IntervalSet, LOG, MixError, createPrivateDir, \
- formatBase64, formatDate, formatFnameTime, formatTime, stringContains
+ formatBase64, formatDate, formatFnameTime, formatTime, previousMidnight, \
+ stringContains
from mixminion.Config import ConfigError
-from mixminion.ServerInfo import ServerInfo, _getDirectoryDigestImpl
+from mixminion.ServerInfo import ServerDirectory, ServerInfo, \
+ _getDirectoryDigestImpl
# Layout:
# basedir
@@ -38,6 +40,8 @@
##Fields: DOCDOC
# baseDir
# serverDir
+ # rejectDir,
+ # archiveDir
# servers (filename->ServerInfo)
# serversByNickname (nickname -> [filename, filename,...])
def __init__(self, baseDir):
@@ -55,7 +59,7 @@
createPrivateDir(self.dirArchiveDir)
self.rescan()
- def importServerInfo(self, server):
+ def importServerInfo(self, server, knownOnly=0):
"DOCDOC"
# Raises ConfigError, MixError,
if stringContains(server, "[Server]"):
@@ -68,17 +72,19 @@
server = ServerInfo(string=contents, assumeValid=0)
nickname = server.getNickname()
- validUntil = server['Server']['Valid-Until']
+ if knownOnly and not self.serversByNickname.has_key(nickname):
+ raise MixError("Unknown server %s: use import-new."%nickname)
+
# Is the server already invalid?
- if validUntil < time.time():
+ if server.isExpiredAt(time.time()):
raise MixError("Descriptor has already expired")
# Is there already a server with the same nickname?
if self.serversByNickname.has_key(nickname):
# Make sure the identity key is the same.
oldServer = self.servers[self.serversByNickname[nickname][0]]
- oldIdentity = oldServer['Server']['Identity']
- newIdentity = server['Server']['Identity']
+ oldIdentity = oldServer.getIdentity()
+ newIdentity = server.getIdentity()
if not pk_same_public_key(newIdentity, oldIdentity):
raise MixError("Identity key has changed for %r" % nickname)
# Okay -- make sure we don't have this same descriptor.
@@ -86,43 +92,53 @@
oldServer = self.servers[fn]
if oldServer['Server']['Digest'] == server['Server']['Digest']:
raise MixError("Server descriptor already inserted.")
+ # Okay -- make sure that this server isn't superseded.
+ if self._serverIsSupersededBy(server,
+ [ self.servers[fn] for fn in self.serversByNickname[nickname]]):
+ raise MixError("Server descriptor is superseded")
- newFile = nickname+"-"+formatFnameTime()
- if os.path.exists(os.path.join(self.serverDir, newFile)):
- idx = 1
- # XXXX This is race-prone if we try to run many insert
- # XXXX processes at once.
- while os.path.exists(os.path.join(self.serverDir,
- "%s.%s"%(newFile,idx))):
- idx += 1
- newFile = "%s.%s" %(newFile,idx)
+ newFile = nickname+"-"+formatFnameTime()
+ f, newFile = _openUnique(os.path.join(self.serverDir, newFile))
+ newFile = os.path.split(newFile)[1]
+ f.write(contents)
+ f.close()
- f = open(os.path.join(self.serverDir, newFile), 'w')
- f.write(contents)
- f.close()
+ # Now update the internal structure
+ self.servers[newFile] = server
+ self.serversByNickname.setdefault(nickname, []).append(newFile)
- # Now update the internal structure
- self.servers[newFile] = server
- self.serversByNickname.setdefault(nickname, []).append(server)
+ def expungeServersByNickname(self, nickname):
+ "DOCDOC"
+ LOG.info("Removing all servers named %s", nickname)
+ if not self.serversByNickname.has_key(nickname):
+ LOG.info(" (No such servers exist)")
+ return
+ servers = self.serversByNickname[nickname]
+ for fn in servers:
+ LOG.info(" Removing %s", fn)
+ os.rename(os.path.join(self.serverDir, fn),
+ os.path.join(self.archiveDir, fn))
+ del self.servers[fn]
+ del self.serversByNickname[nickname]
+ LOG.info(" (%s servers removed)", len(servers))
def generateDirectory(self,
- #XXXX two of the next 4 args are redundant... which?
- startAt, endAt,
- dirValidAfter, dirValidUntil,
- now,
- identityKey):
+ startAt, endAt, extraTime,
+ identityKey, publicationTime=None):
"DOCDOC"
+ if publicationTime is None:
+ publicationTime = time.time()
+ if previousMidnight(startAt) >= previousMidnight(endAt):
+ raise MixError("Validity range does not contain a full day.")
included = []
for fn, s in self.servers.items():
- validAfter = s['Server']['Valid-After']
- validUntil = s['Server']['Valid-Until']
- if validUntil < startAt or endAt < validAfter:
+ if not s.isValidAtPartOf(startAt, endAt+extraTime):
continue
nickname = s.getNickname()
+ validAfter = s['Server']['Valid-After']
included.append((nickname, validAfter, fn))
included.sort()
-
# FFFF We should probably not do all of this in RAM, but what the hey.
# FFFF It will only matter if we have many, many servers in the system.
contents = [ ]
@@ -142,18 +158,34 @@
DirectoryIdentity: %s
DirectoryDigest:
DirectorySignature:
- """ % (formatTime(now), formatDate(dirValidAfter),
- formatDate(dirValidUntil),
+ """ % (formatTime(publicationTime),
+ formatDate(startAt),
+ formatDate(endAt),
formatBase64(pk_encode_public_key(identityKey)))
directory = header+"".join(contents)
directory = _getDirectoryDigestImpl(directory, identityKey)
- for fname in (os.path.join(self.baseDir, "directory"),
- os.path.join(self.dirArchiveDir, "dir-"+formatFnameTime())):
- f = open(fname, 'w')
- f.write(fname)
- f.close()
+ # Make sure that the directory checks out
+ # FFFF remove this once we are _very_ confident.
+ if 1:
+ parsed = ServerDirectory(string=directory)
+ includedDigests = {}
+ for _, _, fn in included:
+ includedDigests[self.servers[fn]['Server']['Digest']] = 1
+ foundDigests = {}
+ for s in parsed.getServers():
+ foundDigests[s['Server']['Digest']] = 1
+ assert foundDigests == includedDigests
+
+ f = open(os.path.join(self.baseDir, "directory"), 'w')
+ f.write(directory)
+ f.close()
+
+ f, _ = _openUnique(os.path.join(self.dirArchiveDir,
+ "dir-"+formatFnameTime()))
+ f.write(directory)
+ f.close()
def getDirectoryFilename(self):
"DOCDOC"
@@ -171,63 +203,52 @@
now = time.time()
removed = {}
- beforeNow = IntervalSet([0, time.time()])
+ beforeNow = IntervalSet([(0, time.time())])
for name, servers in self.serversByNickname.items():
- valid = {}
- published = {}
- for fn in servers:
- s = self.servers[fn]
- published[fn] = s['Server']['Published']
- validAfter = s['Server']['Valid-After']
- validUntil = s['Server']['Valid-Until']
- valid[fn] = IntervalSet([validAfter, validUntil])
-
- for fn in servers:
- vOrig = valid[fn]
- v = vOrig.copy()
- v -= beforeNow
- p = published[fn]
- s = []
- for fn2 in servers:
- if published[fn2] <= p:
- continue
- if vOrig * valid[fn2]:
- v -= valid[fn2]
- s.append(fn2)
- if v.isEmpty():
- LOG.info("Removing superceded descriptor %s", fn)
- LOG.info(" (superceded by %s", ",".join(s))
- removed[fn] = 1
-
- # This is a kinda nasty hack: we never remove the last server for
- # a given nickname. If we did,
- nRemovedByNickname = {}
- for name, fns in self.serversByNickname.items():
- nRemovedByNickname[name] = len(
- [fn for fn in fns if removed.has_key(fn)])
- assert nRemovedByNickname[name] < len(fns)
+ servers = [ (self.servers[fn]['Server']['Published'],
+ fn, self.servers[fn]) for fn in servers ]
+ servers.sort()
+ fns = [ fn for _, fn, _ in servers]
+ servers = [ s for _, _, s in servers ]
+ for idx in range(len(servers)):
+ if self._serverIsSupersededBy(servers[idx],
+ servers[idx+1:]):
+ removed[fns[idx]] = "superceded"
for fn, s in self.servers.items():
if removed.has_key(fn):
continue
- if s['Server']['Valid-Until'] < now - 6000:
- # Don't remove the last key for a nickname.
+ if s.isExpiredAt(now-6000):
+ # The descriptor is expired.
name = s.getNickname()
- if (nRemovedByNickname[name] + 1 ==
- len(self.serversByNickname[name])):
- continue
-
- LOG.info("Removing expired descriptor %s", fn)
- removed[fn] = 1
-
- for fn in removed.keys():
+ removed[fn] = "expired"
+
+ # This is a kinda nasty hack: we never remove the last descriptor for
+ # a given nickname. If we did, we would lose track of the server's
+ # identity key.
+ for name, fns in self.serversByNickname.items():
+ nRemoved = len([fn for fn in fns if removed.has_key(fn)])
+ if nRemoved < len(fns):
+ continue
+ # We're about to remove all the descriptors--that's bad!
+ # We find the most recent one, and remove it from but
+ servers = [ (self.servers[fn]['Server']['Published'],
+ fn, self.servers[fn]) for fn in fns ]
+ servers.sort()
+ fn = servers[-1][1]
+ LOG.info("Retaining %s descriptor %s -- it's the last one for %s",
+ removed[fn], fn, name)
+ del removed[fn]
+
+ for fn, why in removed.items():
+ LOG.info("Removing %s descriptor %s", why, fn)
os.rename(os.path.join(self.serverDir, fn),
os.path.join(self.archiveDir, fn))
-
+
del self.servers[fn]
self.__buildNicknameMap()
-
+
def rescan(self):
"DOCDOC"
self.servers = {}
@@ -244,8 +265,32 @@
self.__buildNicknameMap()
def __buildNicknameMap(self):
+ "DOCDOC"
self.serversByNickname = {}
for fn, server in self.servers.items():
nickname = server.getNickname()
- self.serversByNickname.setDefault(nickname, []).append(fn)
+ self.serversByNickname.setdefault(nickname, []).append(fn)
+
+ def _serverIsSupersededBy(self, server, others):
+ "DOCDOC"
+ validity = server.getIntervalSet()
+ for s in others:
+ if server.isNewerThan(s):
+ continue
+ validity -= s.getIntervalSet()
+ return validity.isEmpty()
+
+def _openUnique(fname):
+ "DOCDOC"
+ base, rest = os.path.split(fname)
+ idx = 0
+ while 1:
+ try:
+ fd = os.open(fname, os.O_WRONLY|os.O_CREAT|os.O_EXCL, 0600)
+ return os.fdopen(fd, 'w'), fname
+ except OSError:
+ pass
+ idx += 1
+ fname = os.path.join(base, "%s.%s"%(rest,idx))
+