[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] More directory work: bulletproof and test votelists; ma...
Update of /home/minion/cvsroot/src/minion/lib/mixminion/directory
In directory moria:/tmp/cvs-serv27457/directory
Modified Files:
DirCGI.py DirMain.py Directory.py ServerInbox.py ServerList.py
Log Message:
More directory work: bulletproof and test votelists; make inboxes use them correctly, etc
Index: DirCGI.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/DirCGI.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- DirCGI.py 10 Jul 2003 23:11:30 -0000 1.8
+++ DirCGI.py 4 Jun 2005 13:55:04 -0000 1.9
@@ -8,6 +8,7 @@
__all__ = [ ]
+# Edit this to the configured value "Homedir" in .mixminion_dir.cf
DIRECTORY_BASE = "/home/nickm/src/MixminionDirectory"
import cgi
Index: DirMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/DirMain.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21
--- DirMain.py 28 Nov 2003 04:14:04 -0000 1.20
+++ DirMain.py 4 Jun 2005 13:55:04 -0000 1.21
@@ -13,7 +13,8 @@
import shutil
import sys
import time
-from mixminion.Common import createPrivateDir, formatTime, LOG, UIError
+from mixminion.Common import createPrivateDir, formatTime, iterFileLines, LOG, \
+ UIError
from mixminion.Config import ConfigError
from mixminion.Crypto import init_crypto, pk_fingerprint, pk_generate, \
pk_PEM_load, pk_PEM_save
@@ -144,7 +145,7 @@
print "No such file %r; skipping" %fn
continue
f = open(fn, 'r')
- for ln in f.readlines():
+ for ln in iterFileLines(f):
ln = ln.strip()
if ln and ln[0] != '#':
badServers.append(ln)
Index: Directory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/Directory.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- Directory.py 3 May 2005 03:26:50 -0000 1.19
+++ Directory.py 4 Jun 2005 13:55:04 -0000 1.20
@@ -8,7 +8,9 @@
__all__ = [ 'ServerList', 'MismatchedID', 'DirectoryConfig', 'Directory' ]
+import binascii
import os
+import re
import stat
import time
@@ -16,7 +18,8 @@
import mixminion.Crypto
from mixminion.Common import LOG, MixError, MixFatalError, UIError, \
- formatBase64, writePickled, readPickled, formatTime
+ formatBase64, iterFileLines, writePickled, readPickled, formatTime
+
class Directory:
"""Wrapper class for directory filestores.
@@ -26,10 +29,6 @@
into the directory, and a 'ServerList' which contains the servers in
the directory along with the directory's private keys and so on.
- A directory server also keeps an 'IDCache' that's readable by the CGI
- user and read/writable by the directory user. It maps nicknames to
- identity keys.
-
The 'ServerInbox' is readable and (mostly) writable by the CGI user.
The 'ServerList' is private, and only readable by the directory user.
(Both of these users must have a group in common.)
@@ -40,7 +39,8 @@
Layout:
BASEDIR/dir [Base for ServerList.]
BASEDIR/inbox [Base for ServerInbox.]
- BASEDIR/identity_cache [File for IDCache.]
+
+ DOCDOC
"""
##Fields:
# config: a DirectoryConfig instance
@@ -166,9 +166,6 @@
"CGIGroup" : ('REQUIRE', None, None),
},
'Directory' : {
- "BadServer" : ("ALLOW*", None, None),
- "BadServerFile" : ("ALLOW*", "filename", None),
- "ExcludeServer" : ("ALLOW*", None, None),
"ClientVersions" : ("REQUIRE", "list", None),
"ServerVersions" : ("REQUIRE", "list", None),
},
@@ -223,127 +220,134 @@
raise mixminion.Config.ConfigError("User %s is not in group %s"
%(cgiuser, cgigrp))
-class VoteFile:
+class VoteFile(mixminion.Filestore.PickleCache):
"""File listing dirserver's current disposition towards various
nickname/identity comibations. Each can be voted 'yes', 'no',
- or 'abstain'.
+ 'abstain', or 'ignore'.
"""
## Fields:
# status: identity fingerprint -> ("yes", "nickname") | ("no", None) |
# ("abstain", None) | ("ignore", None)
- # fname
- # dirty, uid, gid
+ # haveComment: fingerprint -> [ nickname ] for servers in comments.
+ # uid, gid
def __init__(self, fname, uid=None, gid=None):
- self.fname = fname
+ mixminion.Filestore.PickleCache.__init__(
+ self, fname, fname+".cache")
self.uid = uid
- self.gid = gig
- if not self._loadFromCache():
- self._load(fname)
+ self.gid = gid
+ self.status = None
+ self.load()
- def _load(self, fname):
- pat = re.compile(r'(yes|no|abstain|ignore)\s+(\S+)\s+([a-fA-F0-9 ]+)')
- f = open(fname, 'r')
+ def _reload(self,):
+ pat = re.compile(r'(\#?)\s*(yes|no|abstain|ignore)\s+(\S+)\s+([a-fA-F0-9 ]+)')
+ f = open(self._fname_base, 'r')
try:
status = {}
lineof = {}
byName = {}
+ haveComment = {}
lineno = 0
- for line in open(fname, 'r').readlines():
+ fname = self._fname_base
+ for line in iterFileLines(f):
lineno += 1
line = line.strip()
- if not line or line[0] == '#': continue
+ if not line: continue
m = pat.match(line)
if not m:
- LOG.warn("Skipping ill-formed line %s in %s",lineno,fname)
+ if line[0] != '#':
+ LOG.warn("Skipping ill-formed line %s of %s",lineno,fname)
continue
- vote, nickname, fingerprint = m.groups()
+ commented, vote, nickname, fingerprint = m.groups()
try:
mixminion.Config._parseNickname(nickname)
except mixminion.Config.ConfigError, e:
- LOG.warn("Skipping bad nickname '%s', on line %s of %s: %s",
- nickname, lineno, fname, e)
- continue
- try:
- ident = binascii.a2b_hex(fingerprint.replace(" ", ""))
- if len(ident) != mixminion.Crypto.DIGEST_LEN:
- raise TypeError("Wrong length for digest")
- except TypeError, e:
- LOG.warn("Invalid fingerprint on line %s of %s: %s", lineno,
- fname, e)
+ if not commented:
+ LOG.warn("Skipping bad nickname '%s', on line %s of %s: %s",
+ nickname, lineno, fname, e)
continue
- if status.has_key(ident):
- LOG.warn("Ignoring duplicate entry for fingprint on line %s (first appeared on line %s)", lineno, lineof[ident])
+ fingerprint = _normalizeFingerprint(fingerprint)
+ if len(fingerprint) != mixminion.Crypto.DIGEST_LEN * 2:
+ if not commented:
+ LOG.warn("Bad length for digest on line %s of %s",
+ lineno, fname)
+ continue
+ if status.has_key(fingerprint):
+ if not commented:
+ LOG.warn("Ignoring duplicate entry for fingerprint on line %s (first appeared on line %s)", lineno, lineof[fingerprint])
continue
- lineof[ident] = lineno
- if vote == 'yes':
- status[ident] = (vote, nickname)
+ lineof[fingerprint] = lineno
+ if commented:
+ haveComment.setdefault(fingerprint, []).append(
+ nickname.lower())
+ elif vote == 'yes':
+ status[fingerprint] = (vote, nickname)
if byName.has_key(nickname.lower()):
- LOG.warn("Ignoring second yes-vote for a nickname %r",
- nickname)
+ if not commented:
+ LOG.warn("Ignoring second yes-vote for a nickname %r",
+ nickname)
continue
- byName[nickname] = ident
+ byName[nickname.lower()] = fingerprint
else:
- status[ident] = (vote, None)
+ status[fingerprint] = (vote, None)
self.status = status
- self.dirty = 1
+ self.haveComment = haveComment
finally:
f.close()
- def appendUnknownServers(self, lst):
+ def _getForPickle(self):
+ return ("VoteCache-1", self.status, self.haveComment)
+
+ def _setFromPickle(self, p):
+ if not isinstance(p, types.TupleType) or p[0] != 'VoteCache-1':
+ return 0
+ self.status = p[1]
+ self.haveComment = p[2]
+ return 1
+
+ def appendUnknownServers(self, lst, now=None):
# list of [(nickname, fingerprint) ...]
+ lst = [ (name, fp) for name, fp in lst if name.lower() not in
+ self.haveComment.get(_normalizeFingerprint(fp), ()) ]
if not lst:
return
- f = open(fname, 'a+')
+ if now is None:
+ now = time.time()
+ date = formatTime(now,localtime=1)
+ f = open(self._fname_base, 'a+')
try:
f.seek(-1, 2)
nl = (f.read(1) == '\n')
if not nl: f.write("\n")
+ f.write("# Added %s [GMT]:\n"%formatTime(now))
for name, fp in lst:
- f.write("# Added %s\n#abstain %s %s\n"%(date, name, fp))
+ f.write("#abstain %s %s\n"%(name, fp))
+ self.haveComment.setdefault(fp, []).append(
+ binascii.b2a_hex(fp))
finally:
f.close()
- def _loadFromCache(self):
- # raise OSError or return false on can't/shouldn't load.
- cacheFname = self.fname + ".cache"
- try:
- cache_mtime = os.stat(cacheFname)[stat.ST_MTIME]
- file_mtime = os.stat(self.fname)[stat.ST_MTIME]
- except OSError:
- return 0
- if file_mtime >= cache_mtime:
- return 0
- try:
- p = readPickled(cacheFname)
- except (OSError, cPickle.UnpicklingError), _:
- return 0
- if type(p) != types.TupleType or p[0] != 'VoteCache-0':
- return 0
- self.status = p[1]
- self.dirty = 0
- return 1
-
- def saveCache(self):
- cacheFname = self.fname + ".cache"
- writePickled(cacheFname, ("VoteCache-0", self.status), 0640)
+ def save(self):
+ mixminion.Filestore.PickleCache.save(self, 0640)
if self.uid is not None and self.gid is not None:
- _set_uid_gid_mode(cacheFname, self.uid, self.gid, 0640)
- _set_uid_gid_mode(self.name, self.uid, self.gid, 0640)
- self.dirty = 0
+ _set_uid_gid_mode(self._fname_cache, self.uid, self.gid, 0640)
+ _set_uid_gid_mode(self._fname_base, self.uid, self.gid, 0640)
- def getServerStatus(self, server):
- # status + 'unknown' + 'mismatch'
- ident = server.getIdentityDigest()
+ def getStatus(self, fingerprint, nickname):
try:
- vote, nickname = self.status[ident]
+ vote, nick = self.status[_normalizeFingerprint(fingerprint)]
except KeyError:
return "unknown"
- if vote == 'yes' and nickname != server.getNickname():
+ if vote == 'yes' and nickname.lower() != nick.lower():
return "mismatch"
return vote
+ def getServerStatus(self, server):
+ # status + 'unknown' + 'mismatch'
+ return self.getStatus(server.getIdentityFingerprint(),
+ server.getNickname())
+
def _set_uid_gid_mode(fn, uid, gid, mode):
"""Change the permissions on the file named 'fname', so that fname
is owned by user 'uid' and group 'gid', and has permissions 'mode'.
@@ -353,3 +357,6 @@
os.chown(fn, uid, gid)
if (st[stat.ST_MODE] & 0777) != mode:
os.chmod(fn, mode)
+
+def _normalizeFingerprint(fingerprint):
+ return fingerprint.replace(" ", "").upper()
Index: ServerInbox.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerInbox.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- ServerInbox.py 3 May 2005 03:26:50 -0000 1.13
+++ ServerInbox.py 4 Jun 2005 13:55:04 -0000 1.14
@@ -6,7 +6,7 @@
A ServerInbox holds server descriptors received from the outside world
that are not yet ready to be included in the directory. It is designed
to be written to by an untrusted user (e.g., CGI).
- """
+"""
__all__ = [ 'ServerInbox' ]
@@ -21,8 +21,11 @@
world that are not yet ready to be included in the directory.
"""
## Fields:
+ # store: A ServerStore to hold server files. Must be readable/writeable by
+ # directory server user and CGI user.
+ # voteFile: A VoteFile obejct. Must be readable by CGI user.
def __init__(self, store, voteFile):
- """DOCDOC"""
+ """Create a new ServerInbox."""
self.store = store
self.voteFile = voteFile
@@ -44,18 +47,23 @@
now = time.time()
try:
+ #XXXX digest cache??
server = ServerInfo(string=text,assumeValid=0,_keepContents=1)
except MixError, e:
LOG.warn("Rejected invalid server from %s: %s", source,e)
raise UIError("Server descriptor was not valid: %s"%e)
status = self.voteFile.getServerStatus(server)
- if status == "mismatch":#XXXX missing case? (I found an elif here)
- LOG.warn("Rejected server with mismatched identity for %s from %s",
+ if status == "mismatch":
+ LOG.warn("Rejected server with mismatched identity for %r from %s",
nickname, source)
self.store.addServer(server)
raise UIError(("I already know a server named "
"%s with a different key.")%server.getNickname())
+ elif status == "ignore":
+ LOG.warn("Rejected descriptor for ignored server %r from %s",
+ nickname, source)
+ return
if server.isExpiredAt(time.time()):
LOG.warn("Rejecting expired descriptor from %s", source)
@@ -63,7 +71,7 @@
" is probably skewed.")
if status in ("yes", "no", "abstain"):
- LOG.warn("Received update for already-seen server %r from %s (vote=%s)",
+ LOG.info("Received update for server %r from %s (vote=%s)",
server.getNickname(), source, status)
self.store.addServer(server)
return 1
@@ -76,6 +84,9 @@
"Server queued pending manual checking")
def moveEntriesToStore(self, intoStore):
+ """Invoked by directory server. Re-scan elements of the store,
+ moving them into another store 'intoStore' as appropriate.
+ """
keys = self.store.listKeys()
unknown = {}
for k in keys:
@@ -88,7 +99,7 @@
if status not in ("ignore", "mismatch"):
intoStore.addServer(s)
if status == 'unknown':
- unknown[(s.getNickname(), getIdentityFingerprint())]=1
+ unknown[(s.getNickname(), s.getIdentityFingerprint())]=1
self.store.delServer(k)
if unknown:
self.voteFile.appendUnknownServers(unknown.keys())
Index: ServerList.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/directory/ServerList.py,v
retrieving revision 1.57
retrieving revision 1.58
diff -u -d -r1.57 -r1.58
--- ServerList.py 3 May 2005 03:26:50 -0000 1.57
+++ ServerList.py 4 Jun 2005 13:55:04 -0000 1.58
@@ -19,7 +19,6 @@
import os
import time
import threading
-import xreadlines
import mixminion
import mixminion.Config
@@ -30,7 +29,7 @@
pk_same_public_key
from mixminion.Common import IntervalSet, LOG, MixError, MixFatalError, \
UIError, createPrivateDir, formatBase64, formatDate, formatFnameTime, \
- formatTime, Lockfile, openUnique, previousMidnight, readFile, \
+ formatTime, iterFileLines, Lockfile, openUnique, previousMidnight, readFile,\
readPickled, readPossiblyGzippedFile, stringContains, writeFile, \
writePickled
from mixminion.Config import ConfigError
@@ -201,7 +200,7 @@
pass
def loadServer(self, key, keepContents=0, assumeValid=1):
- #XXXX digest-cache
+ #XXXX008 digest-cache
return ServerInfo(fname=os.path.join(self._loc,key),
assumeValid=assumeValid,
_keepContents=keepContents)
@@ -309,7 +308,7 @@
def addServersFromRawDirectoryFile(self, file):
curLines = []
- for line in mixminion.Common.iterFileLines(file):
+ for line in iterFileLines(file):
if line == '[Server]\n' and curLines:
self._addOneFromRawLines(curLines)
del curLines[:]