[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Checkpointing before the great push to get a server wor...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv1509/minion/lib/mixminion
Modified Files:
Common.py Config.py MMTPClient.py MMTPServer.py Queue.py
test.py
Log Message:
Checkpointing before the great push to get a server working.
Added configuration support. Made logs and shred configurable.
Made unexpected closes get handled correctly.
Made shredding work correctly.
Added tests for keyid mismatch.
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- Common.py 9 Jul 2002 04:07:14 -0000 1.8
+++ Common.py 25 Jul 2002 15:52:57 -0000 1.9
@@ -53,12 +53,29 @@
#----------------------------------------------------------------------
# Secure filesystem operations.
#
-# FFFF This needs to be made portable.
-_SHRED_CMD = "/usr/bin/shred"
-if not os.path.exists(_SHRED_CMD):
- warn("%s not found. Files will not be securely deleted.", _SHRED_CMD)
- _SHRED_CMD = None
+_SHRED_CMD = "---"
+_SHRED_OPTS = None
+
+def _shredConfigHook():
+ global _SHRED_CMD
+ global _SHRED_OPTS
+ import mixminion.Config as Config
+ conf = Config.getConfig()
+ cmd, opts = None, None
+ if conf is not None:
+ val = conf['Host'].get('ShredCommand', None)
+ if val is not None:
+ cmd, opts = val
+
+ if cmd is None:
+ if os.path.exists("/usr/bin/shred"):
+ cmd, opts = "/usr/bin/shred/", ["-uz"]
+ else:
+ getLog().warn("Files will not be securely deleted.")
+ cmd, opts = None, None
+
+ _SHRED_CMD, _SHRED_OPTS = cmd, opts
def secureDelete(fnames, blocking=0):
"""Given a list of filenames, removes the contents of all of those
@@ -80,6 +97,14 @@
XXXX The source to shred.c seems to imply that this is harmless, but
XXXX let's try to avoid that, to be on the safe side.
"""
+ if _SHRED_CMD == "---":
+ import mixminion.Config as Config
+ _shredConfigHook()
+ Config.addHook(_shredConfigHook)
+
+ if fnames == []:
+ return
+
if isinstance(fnames, StringType):
fnames = [fnames]
if blocking:
@@ -88,7 +113,7 @@
mode = os.P_NOWAIT
if _SHRED_CMD:
- return os.spawnl(mode, _SHRED_CMD, _SHRED_CMD, "-uz", *fnames)
+ return os.spawnl(mode, _SHRED_CMD, _SHRED_CMD, *(_SHRED_OPTS+fnames))
else:
for f in fnames:
os.unlink(f)
@@ -114,6 +139,7 @@
def reset(self):
if self.file is not None:
self.file.close()
+ # XXXX Fail sanely. :)
self.file = open(self.fname, 'a')
def close(self):
self.file.close()
@@ -134,7 +160,8 @@
'INFO' : 0,
'WARN' : 1,
'ERROR': 2,
- 'FATAL' : 3 }
+ 'FATAL' : 3,
+ 'NEVER' : 100}
class Log:
def __init__(self, minSeverity):
@@ -143,9 +170,31 @@
onReset(self.reset)
onTerminate(self.close)
+ def _configure(self):
+ import mixminion.Config as Config
+ config = Config.getConfig()
+ self.handlers = []
+ if config == None or not config.has_section('Server'):
+ self.setMinSeverity("WARN")
+ self.addHandler(ConsoleLogTarget(sys.stderr))
+ else:
+ self.setMinSeverity(config['Server']['LogLevel'])
+ if config['Server']['EchoMessages']:
+ self.addHandler(ConsoleLogTarget(sys.stderr))
+ logfile = config['Server']['LogFile']
+ if logfile is not None:
+ logfile = os.path.join(config['Server']['Homedir'], "log")
+ self.addHandler(FileLogTarget(logfile))
+
def setMinSeverity(self, minSeverity):
self.severity = _SEVERITIES.get(minSeverity, 1)
+ def getMinSeverity(self):
+ for k,v in _SEVERITIES.items():
+ if v == self.severity:
+ return k
+ assert 0
+
def addHandler(self, handler):
self.handlers.append(handler)
@@ -184,14 +233,13 @@
"""Return the MixMinion log object."""
global _theLog
if _theLog is None:
- # XXXX Configure the log for real
+ import mixminion.Config as Config
_theLog = Log('DEBUG')
- _theLog.addHandler(ConsoleLogTarget(sys.stderr))
+ _theLog._configure()
+ Config.addHook(_theLog._configure)
return _theLog
-
-
#----------------------------------------------------------------------
# Signal handling
@@ -220,6 +268,8 @@
pid, status = os.waitpid(0, 0)
except OSError, e:
break
+ except e:
+ print e, repr(e), e.__class__
def _sigChldHandler(signal_num, _):
'''(Signal handler for SIGCHLD)'''
Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- Config.py 5 Jul 2002 19:51:12 -0000 1.2
+++ Config.py 25 Jul 2002 15:52:57 -0000 1.3
@@ -1,51 +1,218 @@
# Copyright 2002 Nick Mathewson. See LICENSE for licensing information.
# $Id$
+"""Configuration file parsers for Mixminion client and server
+ configuration.
+
+ A configuration file consists of one or more Sections. Each Section
+ has a header and optionally a list of Entries. Each Entry has a key
+ and a value.
+
+ A section header is written as an open bracket, an identifier, and a
+ close bracket. An entry is written as a key, followed optionally by
+ a colon or an equal sign, followed by a value. Values may be split
+ across multiple lines as in RFC822.
+
+ Empty lines are permitted between entries, and between entries and
+ headers. Comments are permitted on lines beginning with a '#'.
+
+ All identifiers are case-sensitive.
+
+ Example:
+
+ [Section1]
+
+ Key1 value1
+ Key2: Value2 value2 value2
+ value2 value2
+ Key3 = value3
+ # A comment
+ Key4=value4
+
+ [Section2]
+ Key5 value5
+ value5 value5 value5
+ """
+
+__all__ = [ 'getConfig', 'loadConfig' ]
+
import re
from cStringIO import StringIO
-from mixminion.Common import MixError
+import mixminion.Common
+from mixminion.Common import MixError, getLog
+import mixminion.Packet
+
+#----------------------------------------------------------------------
+_theConfiguration = None
+
+def loadConfig(fname=0,server=0):
+ """XXXX"""
+ global _theConfiguration
+ assert _theConfiguration is None
+
+ if server:
+ _theConfiguration = ServerConfig(fname)
+ else:
+ _theConfiguration = ClientConfig(fname)
+def getConfig():
+ """XXXX"""
+ return _theConfiguration
+#----------------------------------------------------------------------
+#XXXX
+_CONFIG_HOOKS = []
+def addHook(hook):
+ 'xxxx'
+ _CONFIG_HOOKS.append(hook)
+
+#----------------------------------------------------------------------
+
+# Regular expression to match a section headerr.
_section_re = re.compile(r'\[([^\]]+)\]')
-_entry_re = re.compile(r'([^:= \t]+)[:= \t]\s*(.*)')
-_control_re = re.compile(r'-----(BEGIN|END) ([^-]+)-----')
+
+# Regular expression to match the first line of an entry
+_entry_re = re.compile(r'([^:= \t]+)(?:\s*[:=]|[ \t])\s*(.*)')
class ConfigError(MixError):
+ """Thrown when an error is found in a configuration file."""
pass
-
+
+def _parseBoolean(boolean, validate=0):
+ s = boolean.strip().lower()
+ if s in ("1", "yes", "y", "true", "on"):
+ return 1
+ elif validate and s not in ("0", "no", "n", "false", "off"):
+ raise ConfigError("Invalid boolean %r" % (boolean))
+ else:
+ return 0
+
+def _parseSeverity(severity, validate=0):
+ s = boolean.strip().upper()
+ if validate and not mixminion.Common._SEVERITIES.has_key(s):
+ raise ConfigError("Invalid log level %r" % (severity))
+ return s
+
+def _parseServerMode(mode, validate=0):
+ s = mode.strip().lower()
+ if validate and mode not in ('relay', 'local'):
+ raise ConfigError("Server mode must be 'Relay' or 'Local'")
+ return s
+
+_interval_re = re.compile(r'''(\d+\.?\d*|\.\d+)\s+
+ (second|minute|hour|day|week|month|year)s?''',
+ re.X)
+_seconds_per_unit = {
+ 'second': 1,
+ 'minute': 60,
+ 'hour': 60*60,
+ 'day': 60*60*24,
+ 'week': 60*60*24*7,
+ 'month': 60*60*24*30, # These aren't quite right, but we don't need
+ 'year': 60*60*24*365, # exactness.
+ }
+def _parseInterval(interval, validate=0):
+ inter = interval.strip().lower()
+ m = _interval_re.match(inter)
+ if not m:
+ raise ConfigError("Unrecognized interval %r" % inter)
+ num, unit = float(m.group(1)), m.group(2)
+ nsec = num * _seconds_per_unit[unit]
+ return num, unit, nsec
+
+def _parseInt(integer, validate=0):
+ i = integer.strip().lower()
+ try:
+ return int(i)
+ except ValueError, e:
+ raise ConfigError("Unrecongized integer %r" % (integer))
+
+def _parseIP(ip, validate=0):
+ i = ip.strip().lower()
+ if validate:
+ try:
+ f = mixminion.Packet._packIP(i)
+ except mixminion.Packet.ParseError, p:
+ raise ConfigError("Invalid IP %r" % i)
+
+ return i
+
+def _parseCommand(command, validate=0):
+ c = command.strip().lower().split()
+ if not c:
+ raise ConfigError("Invalid command %r" %command)
+ cmd, opts = c[0], c[1:]
+ if not os.path.exists(cmd) and not os.path.isabs(cmd):
+ for p in os.environ.get('PATH', "").split(os.pathsep):
+ p = os.path.expanduser(p)
+ c = os.path.join(p, cmd)
+ if os.path.exists(c):
+ cmd = c
+ break
+ else:
+ raise ConfigError("No match found for command %r" %cmd)
+ return cmd, opts
+
+def _parseDir(directory, validate=0):
+ d = directory.strip().lower()
+ if not os.path.exists(d):
+ getLog().warn("Trying to create directory %r"%d)
+ try:
+ os.mkdir(d, 0700)
+ except OSError:
+ raise ConfigError("Couldn't create directory %r"%directory)
+ elif not os.path.isdir(d):
+ raise ConfigError("File %r is not a directory"%directory)
+ return d
+
def _parseLine(line):
+ """Helper function. Given a line of a configuration file, return
+ a (TYPE, VALUE) pair, where TYPE is one of the following:
+
+ None: The line is empty or a comment.
+ 'ERR': The line is incorrectly formatted. VALUE is an error message.
+ 'SEC': The line is a section header. VALUE is the section's name.
+ 'ENT': The line is the first line of an entry. VALUE is a (K,V) pair.
+ 'MORE': The line is a continuation line of an entry. VALUE is the
+ contents of the line.
+ """
+
if line == '':
return None, None
- if line.startswith('-----'):
- m = _control_re.match(line)
- if not m:
- return m, "Bad control line"
- return m.group(0), m.group(1)
space = line[0] and line[0] in ' \t'
- line = line.trim()
+ line = line.strip()
if line == '' or line[0] == '#':
return None, None
elif line[0] == '[':
m = _section_re.match(line)
if not m:
return "ERR", "Bad section declaration"
- return m.group(1).trim()
+ return 'SEC', m.group(1).strip()
elif space:
return "MORE", line
else:
m = _entry_re.match(line)
if not m:
return "ERR", "Bad entry"
- return "ENT", m.group(1), m.group(2)
+ return "ENT", (m.group(1), m.group(2))
-def _parseFile(self, file):
- #XXXX What to do with control lines?
+def _parseFile(file):
+ """Helper function. Given an open file object for a configuration
+ file, parse it into sections.
+
+ Returns a list of (SECTION-NAME, SECTION) tuples, where each
+ SECTION is a list of (KEY, VALUE, LINENO) tuples.
+
+ Throws ConfigError if the file is malformatted.
+ """
sections = []
curSection = None
lineno = 0
- for line in f.readlines():
+ lastKey = None
+ for line in file.readlines():
lineno += 1
+ x = _parseLine(line)
type, val = _parseLine(line)
if type == 'ERR':
raise ConfigError("%s at line %s" % (val, lineno))
@@ -54,35 +221,63 @@
sections.append( (val, curSection) )
elif type == 'ENT':
key,val = val
- if not curSection:
+ if curSection is None:
raise ConfigError("Unknown section at line %s" %lineno)
curSection.append( [key, val, lineno] )
lastKey = key
elif type == 'MORE':
if not lastKey:
raise ConfigError("Unexpected indentation at line %s" %lineno)
- curSection[-1][1] = "%s %s" % (curSection[-1][1], line)
+ curSection[-1][1] = "%s %s" % (curSection[-1][1], val)
return sections
def _formatEntry(key,val,w=79,ind=4):
+ """Helper function. Given a key/value pair, returns a NL-terminated
+ entry for inclusion in a configuration file, such that no line is
+ avoidably longer than 'w' characters, and with continuation lines
+ indented by 'ind' spaces.
+ """
ind = " "*ind
if len(val)+len(key)+2 <= 79:
return "%s: %s\n" % (key,val)
- lines = [ "%s: " ]
+ lines = [ "%s: " %key ]
#XXXX Bad implementation.
- for v in " ".split(val):
- if len(lines[-1])+" "+len(v) <= w:
+ for v in val.split(" "):
+ print v
+ if len(lines[-1])+1+len(v) <= w:
lines[-1] = "%s %s" % (lines[-1],v)
else:
lines.append(ind+v)
- lines.append("")
+ lines.append("") # so the last line ends with \n
return "\n".join(lines)
class _ConfigFile:
- # Set in subclass: _syntax is map from sec->{key:
- # ALLOW/REQUIRE/ALLOW*/REQUIRE*/IGNORE}
+ """Base class to parse, validate, and represent configuration files.
+ """
+ ##Fields:
+ # fname: Name of the underlying file. Used by .reload()
+ # _sections: A map from secname->key->value.
+ # _sectionEntries: A map from secname->[ (key, value) ] inorder.
+ # _sectionNames: An inorder list of secnames.
+ # hooks: list of callback functions.
+ #
+ # Set by a subclass:
+ # _syntax is map from sec->{key:
+ # (ALLOW/REQUIRE/ALLOW*/REQUIRE*,
+ # parseFn,
+ # default, ) }
+
+ # A key without a corresponding entry in _syntax gives an error.
+ # A section without a corresponding entry is ignored.
+ # ALLOW* and REQUIRE* permit multiple entries with for a given key:
+ # these entries are read into a list.
+ # The magic key __SECTION__ describes whether a section is requried.
+
def __init__(self, fname=None, string=None):
+ """Create a new _ConfigFile. If fname is set, read from
+ fname. If string is set, parse string."""
+ assert fname is None or string is None
self.fname = fname
if fname:
self.reload()
@@ -96,26 +291,36 @@
self.clear()
def clear(self):
+ """Remove all sections from this _ConfigFile object."""
self._sections = {}
self._sectionEntries = {}
self._sectionNames = []
def reload(self):
+ """Reload this _ConfigFile object from disk. If the object is no
+ longer present and correctly formatted, raise an error, but leave
+ the contents of this object unchanged."""
if not self.fname:
return
- f = open(fname, 'r')
+ f = open(self.fname, 'r')
try:
self.__reload(f)
+ for hook in _CONFIG_HOOKS:
+ hook()
finally:
f.close()
def __reload(self, file):
+ """As in .reload(), but takes an open file object."""
sections = _parseFile(file)
-
+
+ # These will become self.(_sections,_sectionEntries,_sectionNames)
+ # if we are successful.
self_sections = {}
self_sectionEntries = {}
self_sectionNames = []
-
+ sectionEntryLines = {}
+
for secName, secEntries in sections:
self_sectionNames.append(secName)
@@ -124,65 +329,105 @@
section = {}
sectionEntries = []
+ entryLines = []
self_sections[secName] = section
self_sectionEntries[secName] = sectionEntries
+ sectionEntryLines[secName] = entryLines
secConfig = self._syntax.get(secName, None)
if not secConfig:
- #XXXX FFFF
- print "Skipping unrecognized section %s" % (secName)
+ getLog().warn("Skipping unrecognized section %s", secName)
continue
for k,v,line in secEntries:
- sectionEntrties.add( (k,v) )
- rule = secConfig.get(k, None)
+ sectionEntries.append( (k,v) )
+ entryLines.append(line)
+ rule, parseFn, default = secConfig.get(k, (None,None,None))
if not rule:
raise ConfigError("Unrecognized key %s on line %s" %
(k, line))
+ if parseFn is not None:
+ try:
+ v = parseFn(v, validate=1)
+ except ConfigError, e:
+ e.args = ("%s at line %s" %(e.args[0],line))
+ raise e
+
if rule in ('REQUIRE*','ALLOW*'):
if section.has_key(k):
section[k].append(v)
else:
section[k] = [v]
- else: #rule in ('REQUIRE', 'ALLOW')
+ else:
+ assert rule in ('REQUIRE', 'ALLOW')
if section.has_key(k):
raise ConfigError("Duplicate entry for %s at line %s"
% (k, line))
else:
section[k] = v
- for k, rule in secRules:
- if k in ('REQUIRE', 'REQUIRE*') and not section.has_key(k):
+ for k, (rule, parseFn, default) in secConfig.items():
+ if k == '__SECTION__':
+ continue
+ if rule in ('REQUIRE', 'REQUIRE*') and not section.has_key(k):
raise ConfigError("Missing entry %s from section %s"
% (k, secName))
+ elif not section.has_key(k):
+ if parseFn is None:
+ section[k] = default
+ elif default is None:
+ section[k] = default
+ elif rule == 'ALLOW':
+ section[k] = parseFn(default)
+ else:
+ assert rule == 'ALLOW*'
+ section[k] = map(parseFn,default)
- for secName in self._syntax:
- if (secName.get('__SECTION__', 'ALLOW') == 'REQUIRE'
+ for secName, secConfig in self._syntax.items():
+ secRule = secConfig.get('__SECTION__', ('ALLOW',None,None))
+ if (secRule[0] == 'REQUIRE'
and not self_sections.has_key(secName)):
raise ConfigError("Section [%s] not found." %secName)
+ elif not self_sections.has_key(secName):
+ self_sections[secName] = {}
+ self_sectionEntries[secName] = {}
- for secName in self_sectionNames:
+ for s in self_sectionNames:
for k,v in self_sectionEntries[s]:
assert v == self_sections[s][k] or v in self_sections[s][k]
- self.validate(self_sections, self_sectionEntries)
+ self.validate(self_sections, self_sectionEntries, sectionEntryLines)
- self.sections = self_sections
- self.sectionEntries = self_sectionEntries
- self.sectionName = self_sectionNames
+ self._sections = self_sections
+ self._sectionEntries = self_sectionEntries
+ self._sectionNames = self_sectionNames
- def validate(sections, sectionEntries):
+ def validate(self, sections, sectionEntries, entryLines):
+ """Check additional semantic properties of a set of configuration
+ data before overwriting old data. Subclasses should override."""
pass
def __getitem__(self, sec):
+ """self[section] -> dict
+
+ Return a map from keys to values for a given section. If the
+ section was absent, return an empty map."""
return self._sections[sec]
+ def has_section(self, sec):
+ 'XXXX'
+ return self._sections.has_key(sec)
+
def getSectionItems(self, sec):
+ """Return a list of ordered (key,value) tuples for a given section.
+ section was absent, return an empty map."""
return self._sectionEntries[sec]
def __str__(self):
+ """Returns a string configuration file equivalent to this configuration
+ file."""
lines = []
for s in self._sectionNames:
lines.append("[%s]\n"%s)
@@ -192,22 +437,197 @@
return "".join(lines)
-_serverDescriptorSyntax = {
- 'Server' : { 'Descriptor-Version' : 'REQUIRE',
- 'IP' : 'REQUIRE',
- 'Nickname' : 'ALLOW',
- 'Identity' : 'REQUIRE',
- 'Digest' : 'REQUIRE',
- 'Signature' : 'REQUIRE',
- 'Valid-After' : 'REQUIRE',
- 'Valid-Until' : 'REQUIRE',
- 'Contact' : 'ALLOW',
- 'Comments' : 'ALLOW',
- 'Packet-Key' : 'REQUIRE', },
- 'Incoming/MMTP' : { 'MMTP-Decriptor-Version' : 'REQUIRE',
- 'Port' : 'REQUIRE',
- 'Key-Digest' : 'REQUIRE', },
- 'Modules/MMTP' : { 'MMTP-Descriptor-Version' : 'REQUIRE',
- 'Allow' : 'ALLOW*',
- 'Deny' : 'ALLOW*' }
- }
+_interval_re = re.compile('(\d+\.?\d*|\.\d+)\s+(second|minute|hour)s?')
+def validateInterval(s):
+ m = _interval_re.match(s)
+ if not m:
+ return "ERR", "Invalid interval: %r" %s
+
+ return "OK", (float(m.group(1)), m.group(2), )
+
+class ClientConfig(_ConfigFile):
+ _syntax = {
+ 'Host' : { '__SECTION__' : ('REQUIRE', None, None),
+ 'ShredCommand': ('ALLOW', _parseCommand, None),
+ 'EntropySource': ('ALLOW', None, "/dev/urandom"),
+ },
+ 'DirectoryServers' :
+ { '__SECTION__' : ('REQUIRE', None, None),
+ 'ServerURL' : ('ALLOW*', None, None),
+ 'MaxSkew' : ('ALLOW', _parseInterval, "10 minutes") },
+ 'User' : { 'UserDir' : ('ALLOW', _parseDir, "~/.mixminion" ) },
+ 'Security' : { 'PathLength' : ('ALLOW', _parseInt, "8"),
+ 'SURBAddress' : ('ALLOW', None, None),
+ 'SURBPathLength' : ('ALLOW', None, "8") },
+ }
+ def __init__(self, fname=None, string=None):
+ _ConfigFile.__init__(self, fname, string)
+
+ def validate(self, sections, entries, lines):
+ #XXXX Write this
+ pass
+
+ def getShredCommand(self):
+ return self['Host'].get('ShredCommand', None)
+
+ def getEntropySource(self):
+ return self['Host'].get('EntropySource', None)
+
+ def getDirectoryServerURLs(self):
+ return self['DirectoryServers'].get('ServerURL', [])
+
+ def getMaxSkew(self):
+ # returns seconds.
+ skew = self['DirectoryServers'].get('MaxSkew', "10 minutes")
+ _, _, nsec = _parseInterval(skew)
+ return nsec
+
+ def getUserDir(self):
+ return self['UserDir'].get('UserDir', '.minion')
+
+ def getPathLength(self):
+ return int(self['Security'].get('PathLength', '8'))
+
+ def getSURBPathLength(self):
+ return int(self['Security'].get('SURBPathLength', '8'))
+
+ def getSURBAddress(self):
+ return self['Security'].get('SURBAddress', None)
+
+class ServerConfig(_ConfigFile):
+ _syntax = {
+ 'Host' : ClientConfig._syntax['Host'],
+ 'Server' : { '__SECTION__' : ('REQUIRE', None, None),
+ 'Homedir' : ('ALLOW', _parseDir, "/var/spool/minion"),
+ 'LogFile' : ('ALLOW', None, None),
+ 'LogLevel' : ('ALLOW', _parseSeverity, "WARN"),
+ 'EchoMessages' : ('ALLOW', _parseBoolean, "no"),
+ 'EncryptIdentityKey' : ('REQUIRE', _parseBoolean, "yes"),
+ 'PublicKeyLifetime' : ('REQUIRE', _parseInterval,
+ "30 days"),
+ 'EncryptPublicKey' : ('REQUIRE', _parseBoolean, "no"),
+ 'Mode' : ('REQUIRE', _parseServerMode, "local"),
+ },
+ 'DirectoryServers' : { 'ServerURL' : ('ALLOW*', None, None),
+ 'Publish' : ('ALLOW', _parseBoolean, "no"),
+ 'MaxSkew' : ('ALLOW', _parseInterval,
+ "10 minutes",) },
+ 'Incoming/MMTP' : { 'Enabled' : ('REQUIRE', _parseBoolean, "no"),
+ 'IP' : ('ALLOW', _parseIP, None),
+ 'Port' : ('ALLOW', _parseInt, "48099"),
+ 'Allow' : ('ALLOW*', None, None),
+ 'Deny' : ('ALLOW*', None, None) },
+ 'Outgoing/MMTP' : { 'Enabled' : ('REQUIRE', _parseBoolean, "no"),
+ 'Allow' : ('ALLOW*', None, None),
+ 'Deny' : ('ALLOW', None, None) },
+ 'Delivery/MBox' : { 'Enabled' : ('REQUIRE', _parseBoolean, "no"),
+ 'AddressFile' : ('REQUIRE', None, None),
+ 'Command' : ('ALLOW', _parseCommand, "sendmail") },
+ }
+ # Missing: Queue-Size / config options
+ # timeout
+ def __init__(self, fname=None, string=None):
+ _ConfigFile.__init__(self, fname, string)
+
+ def validate(self, sections, entries, lines):
+ #XXXX write this.
+ pass
+
+ def getShredCommand(self):
+ return self['Host'].get('ShredCommand', None)
+
+ def getEntropySource(self):
+ return self['Host'].get('EntropySource', None)
+
+ def getHomeDir(self):
+ return self['Server'].get('Homedir', "/var/spool/minion")
+
+ def getLogLevel(self):
+ return _parseSeverity(self['Server'].get('LogLevel', "WARN"))
+
+ def getLogFile(self):
+ return self['Server'].get('LogFile', None)
+
+ def getEchoMessages(self):
+ return _parseBoolean(self['Server'].get('EchoMessages', "no"))
+
+ def getEncryptIdentityKey(self):
+ return _parseBoolean(self['Server'].get('EncryptIdentityKey', "yes"))
+
+ def getMode(self):
+ return _parseServerMode(self['Server'].get('Mode', "local"))
+
+ def getPublicKeyLifetime(self):
+ _, _, nsec =_parseInterval(self['Server'].get('PublicKeyLifetime',
+ '1 month'))
+ return nsec
+
+ def getEncryptPublicKey(self):
+ return _parseBoolean(self['Server'].get('EncryptPublicKey', "no"))
+
+ def getPublish(self):
+ return _parseBoolean(self['DirectoryServers'].get('Publish', "no"))
+
+ def getDirectoryServerURLs(self):
+ return self['DirectoryServers'].get('ServerURL', [])
+
+ def getMaxSkew(self):
+ # returns seconds.
+ skew = self['DirectoryServers'].get('MaxSkew', "10 minutes")
+ _, _, nsec = _parseInterval(skew)
+ return nsec
+
+ def getIncomingMMTPEnabled(self):
+ return _parseBoolean(self['Incoming/MMTP'].get('Enabled', 'no'))
+
+ def getIncomingMMTP_IP(self):
+ return _parseIP(self['Incoming/MMTP'].get('IP', None))
+
+ def getIncomingMMTP_Port(self):
+ return _parseInt(self['Incoming/MMTP'].get('Port', None))
+
+ def getIncomingMMTP_Rules(self):
+ #XXXX WRITE ME
+ pass
+
+ def getOutgoingMMTPEnabled(self):
+ return _parseBoolean(self['Outgoing/MMTP'].get('Enabled', 'no'))
+
+ def getOutgoingMMTPRules(self):
+ #XXXX WRITE ME
+ pass
+
+ def getMBoxEnabled(self):
+ #XXXX WRITE ME
+ pass
+
+ def getMBoxEnabled(self):
+ return _parseBoolean(self['Delivery/MBox'].get('Enabled', 'no'))
+
+ def getMBoxAddressFile(self):
+ return self['Delivery/MBox'].get('AddressFile', None)
+
+ def getMBoxCommand(self):
+ return self['Delivery/MBox'].get('Command', None)
+
+## _serverDescriptorSyntax = {
+## 'Server' : { 'Descriptor-Version' : 'REQUIRE',
+## 'IP' : 'REQUIRE',
+## 'Nickname' : 'ALLOW',
+## 'Identity' : 'REQUIRE',
+## 'Digest' : 'REQUIRE',
+## 'Signature' : 'REQUIRE',
+## 'Valid-After' : 'REQUIRE',
+## 'Valid-Until' : 'REQUIRE',
+## 'Contact' : 'ALLOW',
+## 'Comments' : 'ALLOW',
+## 'Packet-Key' : 'REQUIRE', },
+## 'Incoming/MMTP' : { 'MMTP-Descriptor-Version' : 'REQUIRE',
+## 'Port' : 'REQUIRE',
+## 'Key-Digest' : 'REQUIRE', },
+## 'Modules/MMTP' : { 'MMTP-Descriptor-Version' : 'REQUIRE',
+## 'Allow' : 'ALLOW*',
+## 'Deny' : 'ALLOW*' }
+## }
+
+
Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- MMTPClient.py 9 Jul 2002 04:07:14 -0000 1.4
+++ MMTPClient.py 25 Jul 2002 15:52:57 -0000 1.5
@@ -76,7 +76,9 @@
def sendMessages(targetIP, targetPort, targetKeyID, packetList):
"""Sends a list of messages to a server."""
con = BlockingClientConnection(targetIP, targetPort, targetKeyID)
- con.connect()
- for p in packetList:
- con.sendPacket(p)
- con.shutdown()
+ try:
+ con.connect()
+ for p in packetList:
+ con.sendPacket(p)
+ finally:
+ con.shutdown()
Index: MMTPServer.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPServer.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- MMTPServer.py 9 Jul 2002 04:07:14 -0000 1.6
+++ MMTPServer.py 25 Jul 2002 15:52:57 -0000 1.7
@@ -357,6 +357,11 @@
self.__server.registerWriter(self)
except _ml.TLSWantRead:
self.__server.registerReader(self)
+ except _ml.TLSClosed:
+ warn("Unexpectedly closed connection")
+
+ self.__sock.close()
+ self.__server.unregister(self)
except _ml.TLSError:
if self.__state != self.__shutdownFn:
warn("Unexpected error: closing connection.")
Index: Queue.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Queue.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- Queue.py 9 Jul 2002 04:07:14 -0000 1.5
+++ Queue.py 25 Jul 2002 15:52:57 -0000 1.6
@@ -61,6 +61,9 @@
'create' is true, creates the directory if necessary. If 'scrub'
is true, removes any incomplete or invalidated messages from the
Queue."""
+
+ secureDelete([]) # Make sure secureDelete is configured. HACK!
+
self.rng = AESCounterPRNG()
self.dir = location
@@ -233,12 +236,17 @@
return base64.encodestring(junk).strip().replace("/","-")
def _secureDelete_bg(files, cleanFile):
- if os.fork() != 0:
- return
+ pid = os.fork()
+ if pid != 0:
+ return pid
# Now we're in the child process.
- secureDelete(files, blocking=1)
+ try:
+ secureDelete(files, blocking=1)
+ except OSError, e:
+ # This is sometimes thrown when shred finishes before waitpid.
+ pass
try:
os.unlink(cleanFile)
- except OSError:
+ except OSError, e:
pass
os._exit(0)
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- test.py 9 Jul 2002 04:07:14 -0000 1.11
+++ test.py 25 Jul 2002 15:52:57 -0000 1.12
@@ -21,7 +21,7 @@
import tempfile
import types
-from mixminion.Common import MixError, MixFatalError, getLog
+from mixminion.Common import MixError, MixFatalError, MixProtocolError, getLog
try:
import unittest
@@ -1229,17 +1229,20 @@
queue.removeAll()
def testQueueOps(self):
- #XXXX COMMENT ME
queue1 = Queue(self.d2, create=1)
queue2 = Queue(self.d3, create=1)
+ # Put 100 messages in queue1
handles = [queue1.queueMessage("Sample message %s" % i)
for i in range(100)]
hdict = {}
for i in range(100): hdict[handles[i]] = i
+ # Make sure that queue1 has all 100 elements
self.assertEquals(queue1.count(), 100)
self.assertEquals(len(handles), 100)
+ # Get the messages in random order, and make sure the contents
+ # of each one are correct
foundHandles = queue1.pickRandom(100)
self.assertEquals(len(foundHandles), 100)
for h in foundHandles:
@@ -1250,38 +1253,46 @@
assert len(hdict) == len(handles) == 100
+ # Move the first 30 messages to queue2
q2h = []
for h in handles[:30]:
nh = queue1.moveMessage(h, queue2)
q2h.append(nh)
+ # Look at the messages in queue2, 15 then 30 at a time.
from string import atoi
- seen = {}
- for h in queue2.pickRandom(30):
- c = queue2.messageContents(h)
- self.failUnless(c.startswith("Sample message "))
- i = atoi(c[15:])
- self.failIf(seen.has_key(i))
- seen[i]=1
+ for group in queue2.pickRandom(15), queue2.pickRandom(30):
+ seen = {}
+ for h in group:
+ c = queue2.messageContents(h)
+ self.failUnless(c.startswith("Sample message "))
+ i = atoi(c[15:])
+ self.failIf(seen.has_key(i))
+ seen[i]=1
+ # Make sure that we got all 30 messages
for i in range(30):
self.failUnless(seen.has_key(i))
+ # Remove messages 30..59 from queue1.
for h in handles[30:60]:
queue1.removeMessage(h)
-
self.assertEquals(40, queue1.count())
+
+ # Make sure that smaller pickRandoms work.
L1 = queue1.pickRandom(10)
L2 = queue1.pickRandom(10)
self.failUnless(len(L1) == 10)
self.failUnless(len(L2) == 10)
self.failUnless(L1 != L2)
+ # Test 'openMessage'
f = queue1.openMessage(handles[60])
s = f.read()
f.close()
self.assertEquals(s, "Sample message 60")
+ # test successful 'openNewMessage'
f, h = queue1.openNewMessage()
f.write("z"*100)
self.failUnlessRaises(IOError, queue1.messageContents, h)
@@ -1290,6 +1301,7 @@
self.assertEquals(queue1.messageContents(h), "z"*100)
self.assertEquals(queue1.count(), 41)
+ # test aborted 'openNewMessage'
f, h = queue1.openNewMessage()
f.write("z"*100)
queue1.abortMessage(f,h)
@@ -1297,9 +1309,9 @@
self.assertEquals(queue1.count(), 41)
self.assert_(not os.path.exists(os.path.join(self.d2, "msg_"+h)))
+ # Scrub both queues.
queue1.removeAll()
queue2.removeAll()
-
queue1.cleanQueue()
queue2.cleanQueue()
@@ -1309,6 +1321,7 @@
#----------------------------------------------------------------------
# MMTP
+# XXXX Write more tests
import mixminion.MMTPServer
import mixminion.MMTPClient
@@ -1380,7 +1393,7 @@
self.server.process(0.1)
count = count + 1
- def testBlockingTransmission(self):
+ def ___testBlockingTransmission(self):
self.doTest(self._testBlockingTransmission)
def testNonblockingTransmission(self):
@@ -1407,6 +1420,18 @@
self.failUnless(messagesIn == messages)
+ # Now, with bad keyid.
+ t = threading.Thread(None,
+ self.failUnlessRaises,
+ args=(MixProtocolError,
+ mixminion.MMTPClient.sendMessages,
+ "127.0.0.1", TEST_PORT, "Z"*20, messages))
+ t.start()
+ while t.isAlive():
+ server.process(0.1)
+ t.join()
+
+
def _testNonblockingTransmission(self):
server, listener, messagesIn, keyid = _getMMTPServer()
self.listener = listener
@@ -1433,17 +1458,150 @@
self.assertEquals(len(messagesIn), len(messages))
self.failUnless(messagesIn == messages)
-
+
+ # Again, with bad keyid.
+ clientcon = mixminion.MMTPServer.MMTPClientConnection(
+ _getTLSContext(0), "127.0.0.1", TEST_PORT, "Z"*20,
+ messages[:], None)
+ clientcon.register(async)
+ def clientThread(clientcon=clientcon, async=async):
+ while not clientcon.isShutdown():
+ async.process(2)
+
+
+
+ severity = getLog().getMinSeverity()
+ getLog().setMinSeverity("ERROR") #suppress warning
+ try:
+ server.process(0.1)
+ t = threading.Thread(None, clientThread)
+
+ t.start()
+ while t.isAlive():
+ server.process(0.1)
+ t.join()
+ finally:
+ getLog().setMinSeverity(severity) #unsuppress
+
#----------------------------------------------------------------------
+# Config files
+
+from mixminion.Config import _ConfigFile, ConfigError
+
+class TestConfigFile(_ConfigFile):
+ _syntax = { 'Sec1' : {'__SECTION__': ('REQUIRE', None, None),
+ 'Foo': ('REQUIRE', None, None),
+ 'Bar': ('ALLOW', None, None),
+ 'Baz': ('ALLOW', None, None),},
+ 'Sec2' : {'Fob': ('ALLOW*', None, None),
+ 'Bap': ('REQUIRE', None, None),
+ 'Quz': ('REQUIRE*', None, None), }
+ }
+ def __init__(self, fname=None, string=None):
+ _ConfigFile.__init__(self,fname,string)
+
+class ConfigFileTests(unittest.TestCase):
+ def testValidFiles(self):
+ TCF = TestConfigFile
+ shorterString = """[Sec1]\nFoo a\n"""
+ f = TCF(string=shorterString)
+ self.assertEquals(f['Sec1']['Foo'], 'a')
+ f = TCF(string="""\n\n [ Sec1 ] \n \n\nFoo a \n""")
+ self.assertEquals(f['Sec1']['Foo'], 'a')
+ self.assertEquals(f['Sec2'], {})
+
+ longerString = """[Sec1]
+
+Foo= abcde f
+
+Bar bar
+Baz:
+ baz
+ and more baz
+ and more baz
+[Sec2]
+
+# Comment
+Bap +
+Quz 99 99
+
+
+Fob=1
+Quz : 88
+ 88
+
+ """
+
+ f = TCF(string=longerString)
+ self.assertEquals(f['Sec1']['Foo'], 'abcde f')
+ self.assertEquals(f['Sec1']['Bar'], 'bar')
+ self.assertEquals(f['Sec1']['Baz'], ' baz and more baz and more baz')
+ self.assertEquals(f['Sec2']['Bap'], '+')
+ self.assertEquals(f['Sec2']['Fob'], ['1'])
+ self.assertEquals(f['Sec2']['Quz'], ['99 99', '88 88'])
+ self.assertEquals(f.getSectionItems('Sec2'),
+ [ ('Bap', '+'),
+ ('Quz', '99 99'),
+ ('Fob', '1'),
+ ('Quz', '88 88') ])
+
+ self.assertEquals(str(f),
+ ("[Sec1]\nFoo: abcde f\nBar: bar\nBaz: baz and more baz"+
+ " and more baz\n\n[Sec2]\nBap: +\nQuz: 99 99\nFob: 1\n"+
+ "Quz: 88 88\n\n"))
+ # Test file input
+ fn = tempfile.mktemp()
+ unlink_on_exit(fn)
+
+ file = open(fn, 'w')
+ file.write(longerString)
+ file.close()
+ f = TCF(fname=fn)
+ self.assertEquals(f['Sec1']['Bar'], 'bar')
+ self.assertEquals(f['Sec2']['Quz'], ['99 99', '88 88'])
+
+ # Test failing reload
+ file = open(fn, 'w')
+ file.write("[Sec1]\nFoo=99\nBadEntry 3\n\n")
+ file.close()
+ self.failUnlessRaises(ConfigError, f.reload)
+ self.assertEquals(f['Sec1']['Foo'], 'abcde f')
+ self.assertEquals(f['Sec1']['Bar'], 'bar')
+ self.assertEquals(f['Sec2']['Quz'], ['99 99', '88 88'])
+
+
+ # Test 'reload' operation
+ file = open(fn, 'w')
+ file.write(shorterString)
+ file.close()
+ f.reload()
+ self.assertEquals(f['Sec1']['Foo'], 'a')
+ self.assertEquals(f['Sec1'].get('Bar', None), None)
+ self.assertEquals(f['Sec2'], {})
+
+ def testBadFiles(self):
+ TCF = TestConfigFile
+ def fails(string, self=self):
+ self.failUnlessRaises(ConfigError, TestConfigFile, None, string)
+
+ fails("Foo = Bar\n")
+ fails("[Sec1]\n Foo = Bar\n")
+ fails("[Sec1]\nFoo! Bar\n")
+
+ fails("[Sec1]\nFoob: Bar\n") # No such key
+ fails("[Sec1]\nFoo: Bar\nFoo: Bar\n") # Duplicate key
+ fails("[Sec1]\nBaz: 3\n") # Missing key
+ fails("[Sec2]\nBap = 9\nQuz=6\n") # Missing section
+ fails("[Sec1]\nFoo 1\n[Sec2]\nBap = 9\n") # Missing require*
def testSuite():
suite = unittest.TestSuite()
loader = unittest.TestLoader()
tc = loader.loadTestsFromTestCase
- getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
suite.addTest(tc(MinionlibCryptoTests))
suite.addTest(tc(CryptoTests))
suite.addTest(tc(FormatTests))
+ suite.addTest(tc(ConfigFileTests))
suite.addTest(tc(HashLogTests))
suite.addTest(tc(BuildMessageTests))
suite.addTest(tc(PacketHandlerTests))
@@ -1452,6 +1610,10 @@
return suite
def testAll():
+ # Disable TRACE and DEBUG log messages, unless somebody overrides from
+ # the environment.
+ getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
+
unittest.TextTestRunner().run(testSuite())
if __name__ == '__main__':