[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Bulletproof make process; add generic CLI code
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.seul.org:/tmp/cvs-serv12164/lib/mixminion
Modified Files:
Modules.py benchmark.py test.py
Added Files:
Main.py testSupport.py
Log Message:
Bulletproof make process; add generic CLI code
HACKING:
Improve accuracy of build instructions
Makefile:
- Search for the right python command *very* carefully
- Detect missing ./contrib/openssl/libcrypto.a
- Add 'download-openssl', 'build-openssl', and 'install-openssl'
targets. (Can't test download-openssl yet, since it openssl.org
seems to be down at the moment.)
setup.py:
- Remove old ssl related stuff
Modules.py:
- Rename classes
Main.py:
- New file for generic CLI code to set up PYTHONPATH and launch
other modules.
testSupport:
- New file for shhared code between test.py and benchmark.py
- Add beginnings of simplified 'testing' module
- Move mix_mktemp into testSupport
benchmark.py, test.py:
- Add new CLI logic
- Remove straggling unlink/rmdir calls.
--- NEW FILE: Main.py ---
#!/usr/bin/python2
"""Code to correct the python path, and multiplex between the various
Mixminion CLIs.
This is the command-line entry point for all of mixminion.
"""
import sys
import stat
import os
def filesAreSame(f1, f2):
"""Return true if f1 and f2 are exactly the same file."""
if os.path.normpath(f1) == os.path.normpath(f2):
return 1
try:
ino1 = os.stat(f1)[stat.ST_INO]
ino2 = os.stat(f2)[stat.ST_INO]
return ino1 and ino1 > 0 and ino1 == ino2
except OSError, e:
return 0
def correctPath(myself):
"""Given a command file (taken from sys.argv[0]), try to adjust sys.path
so that 'import mixminion' will work.
(If the admin uses distutils to install Mixminion, the code will
wind up somewhere appropriate on pythonpath. This isn't good enough,
however: we want to run even when sysadmins don't understand distutils.)
"""
import os
orig_cmd = myself
# First, resolve all links.
while os.path.islink(myself):
f = os.readlink(myself)
# Now, the module ought to be living in x/y/z/mixminon/Foo.py.
# The "x/y/z" is the part we're interested in.
mydir = os.path.split(myself)[0]
parentdir, miniondir = os.path.split(mydir)
if not miniondir == 'mixminion':
print >>sys.stderrr, ("Bad mixminion installation:\n"+
" I resolved %s to %s, but expected to find ../mixminion/Main.py")%(
orig_cmd, myself)
# Now we check whether there's already an entry in sys.path. If not,
# we add the directory we found.
parentdir = os.path.normpath(parentdir)
foundEntry = 0
for pathEntry in sys.path:
if os.path.normpath(pathEntry) == parentdir:
foundEntry = 1; break
ent = os.path.join(pathEntry, 'mixminion', 'Main.py')
if os.path.exists(ent) and filesAreSame(pathEntry, myself):
foundEntry = 1; break
if not foundEntry:
print >>sys.stderr, "Adding %s to PYTHONPATH" % parentdir
sys.path[0:0] = [ parentdir ]
# Finally, we make sure it all works.
try:
import mixminion.Main
except ImportError, e:
print >>sys.stderr,"Unable to find correct path for mixminion."
sys.exit(1)
# Global map from command name to 2-tuples of (module_name, function_name).
#
# 'Main.py <cmd> arg1 arg2 arg3' will result in a call to function_name
# in module_name. The function should take two arguments: a string to
# be used as command name in error messages, and a list of [arg1,arg2,arg3].
_COMMANDS = {
"unittests" : ( 'mixminion.test', 'testAll' ),
"benchmarks" : ( 'mixminion.benchmark', 'timeAll' ),
"server" : ( 'mixminion.ServerMain', 'runServer' )
}
def main(args):
"""Given a list of strings in the same format as sys.argv, use args[0]
to correct sys.path; use args[1] to pick a command from _COMMANDS, and
use args[2:] as arguments.
"""
correctPath(args[0])
# Check whether we have a recognized command.
if len(args) == 1 or not _COMMANDS.has_key(args[1]):
# FFFF we could do better in generating a usage message here.
cmds = _COMMANDS.keys()
cmds.sort()
print >>sys.stderr, "Usage: %s {%s} [arguments]" %(
args[0], "|".join(cmds))
sys.exit(1)
# Read the module and function.
command_module, command_fn = _COMMANDS[args[1]]
mod = __import__(command_module, {}, {}, [command_fn])
func = getattr(mod, command_fn)
# Invoke the command.
func(" ".join(args[0:2]), args[2:])
if __name__ == '__main__':
main(sys.argv)
--- NEW FILE: testSupport.py ---
# Copyright 2002 Nick Mathewson. See LICENSE for licensing information.
# $Id: testSupport.py,v 1.1 2002/08/25 03:48:48 nickm Exp $
"""mixminion.testSupport
Shared support code for unit tests, benchmark tests, and integration tests
"""
import os
import sys
import stat
from mixminion.Common import waitForChildren
from mixminion.Config import _parseBoolean, ConfigError
from mixminion.Modules import DeliveryModule, ImmediateDeliveryQueue, \
SimpleModuleDeliveryQueue, DELIVER_OK, DELIVER_FAIL_RETRY, \
DELIVER_FAIL_NORETRY
def DirectoryDumpModule(DeliveryModule):
def getConfigSyntax(self):
return { 'Testing/DirectoryDump':
{ 'Location' : ('REQUIRE', None, None),
'UseQueue': ('REQUIRE', _parseBoolean, None) } }
def validateConfig(self, sections, entries, lines, contents):
loc = sections['Testing/DirectoryDump'].get('Location')
if loc and not os.path.isdir(loc):
raise ConfigError("Directory does not exist: %r"%loc)
def configure(self, config, manager):
self.loc = sections['Testing/DirectoryDump'].get('Location')
if not self.loc:
return
self.useQueue = sections['Testing/DirectoryDump']['UseQueue']
manager.registerModule(self)
max = -1
for f in os.listdir(self.loc):
if int(f) > max:
max = int(f)
self.next = max+1
def getServerInfoBlock(self):
return ""
def getName(self):
return "Testing_DirectoryDump"
def getExitTypes(self):
return [ 0xFFFE ]
def createDeliveryQueue(self, queueDir):
if self.useQueue:
return SimpleModuleDeliveryQueue()#XXXX
else:
return ImmediateDeliveryQueue(self)
def processMessage(self, message, exitType, exitInfo):
assert exitType == 0xFFFE
if exitInfo == 'fail':
return DELIVER_FAIL_RETRY
elif exitIno == 'FAIL!':
return DELIVER_FAIL_NORETRY
f = open(os.path.join(self.loc, self.next), 'w')
self.next += 1
f.write(exitInfo)
f.write("\n")
f.write(message)
f.close()
return DELIVER_OK
#----------------------------------------------------------------------
# Test for acceptable permissions and uid on directory?
_MM_TESTING_TEMPDIR_PARANOIA = 1
# Holds
_MM_TESTING_TEMPDIR = None
_MM_TESTING_TEMPDIR_COUNTER = 0
_MM_TESTING_TEMPDIR_REMOVE_ON_EXIT = 1
def mix_mktemp(extra=""):
'''mktemp wrapper. puts all files under a securely mktemped
directory.'''
global _MM_TESTING_TEMPDIR
global _MM_TESTING_TEMPDIR_COUNTER
if _MM_TESTING_TEMPDIR is None:
import tempfile
temp = tempfile.mktemp()
paranoia = _MM_TESTING_TEMPDIR_PARANOIA
if paranoia and os.path.exists(temp):
print "I think somebody's trying to exploit mktemp."
sys.exit(1)
try:
os.mkdir(temp, 0700)
except OSError, e:
print "Something's up with mktemp: %s" % e
sys.exit(1)
if not os.path.exists(temp):
print "Couldn't create temp dir %r" %temp
sys.exit(1)
st = os.stat(temp)
if paranoia:
if st[stat.ST_MODE] & 077:
print "Couldn't make temp dir %r with secure permissions" %temp
sys.exit(1)
if st[stat.ST_UID] != os.getuid():
print "The wrong user owns temp dir %r"%temp
sys.exit(1)
parent = temp
while 1:
p = os.path.split(parent)[0]
if parent == p:
break
parent = p
st = os.stat(parent)
m = st[stat.ST_MODE]
if m & 02 and not (m & stat.S_ISVTX):
print "Directory %s has fishy permissions %o" %(parent,m)
sys.exit(1)
if st[stat.ST_UID] not in (0, os.getuid()):
print "Directory %s has bad owner %s" % st[stat.UID]
sys.exit(1)
_MM_TESTING_TEMPDIR = temp
if _MM_TESTING_TEMPDIR_REMOVE_ON_EXIT:
import atexit
atexit.register(deltree, temp)
_MM_TESTING_TEMPDIR_COUNTER += 1
return os.path.join(_MM_TESTING_TEMPDIR,
"tmp%05d%s" % (_MM_TESTING_TEMPDIR_COUNTER,extra))
_WAIT_FOR_KIDS = 1
def deltree(*dirs):
global _WAIT_FOR_KIDS
if _WAIT_FOR_KIDS:
print "Waiting for shred processes to finish."
waitForChildren()
_WAIT_FOR_KIDS = 0
for d in dirs:
if os.path.isdir(d):
for fn in os.listdir(d):
loc = os.path.join(d,fn)
if os.path.isdir(loc):
deltree(loc)
else:
os.unlink(loc)
os.rmdir(d)
elif os.path.exists(d):
os.unlink(d)
Index: Modules.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Modules.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- Modules.py 21 Aug 2002 19:09:48 -0000 1.7
+++ Modules.py 25 Aug 2002 03:48:48 -0000 1.8
@@ -85,7 +85,7 @@
via this module. The default implementation returns a
SimpleModuleDeliveryQueue, which (though adequate) doesn't
batch messages intended for the same destination."""
- return _SimpleModuleDeliveryQueue(self, queueDir)
+ return SimpleModuleDeliveryQueue(self, queueDir)
def processMessage(self, message, exitType, exitInfo):
"""Given a message with a given exitType and exitInfo, try to deliver
@@ -96,7 +96,7 @@
DELIVER_FAIL_NORETRY (if the message shouldn't be tried later)."""
raise NotImplementedError("processMessage")
-class _ImmediateDeliveryQueue:
+class ImmediateDeliveryQueue:
"""Helper class usable as delivery queue for modules that don't
actually want a queue. Such modules should have very speedy
processMessage() methods, and should never have deliery fail."""
@@ -120,7 +120,7 @@
# We do nothing here; we already delivered the messages
pass
-class _SimpleModuleDeliveryQueue(mixminion.Queue.DeliveryQueue):
+class SimpleModuleDeliveryQueue(mixminion.Queue.DeliveryQueue):
"""Helper class used as a default delivery queue for modules that
don't care about batching messages to like addresses."""
def __init__(self, module, directory):
@@ -291,7 +291,7 @@
def getExitTypes(self):
return [ DROP_TYPE ]
def createDeliveryQueue(self, directory):
- return _ImmediateDeliveryQueue(self)
+ return ImmediateDeliveryQueue(self)
def processMessage(self, message, exitType, exitInfo):
getLog().debug("Dropping padding message")
return DELIVER_OK
Index: benchmark.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/benchmark.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- benchmark.py 19 Aug 2002 15:33:56 -0000 1.8
+++ benchmark.py 25 Aug 2002 03:48:48 -0000 1.9
@@ -12,7 +12,7 @@
"""
from time import time
-from mixminion.test import mix_mktemp
+from mixminion.testSupport import mix_mktemp
__pychecker__ = 'no-funcdoc no-reimport'
__all__ = [ 'timeAll', 'testLeaks1', 'testLeaks2' ]
@@ -246,14 +246,18 @@
def _hashlogTiming(fname, load):
from mixminion.HashLog import HashLog
- prng = AESCounterPRNG("a"*16)
+ prng = AESCounterPRNG("a"*16)
+
+ print "Testing hash log (%s entries)"%load
+ if load > 20000:
+ print "This may take a few minutes..."
h = HashLog(fname, "A")
hashes = [ prng.getBytes(20) for _ in xrange(load) ]
t = time()
- for hash in hashes:
- h.logHash(hash)
+ for n in xrange(len(hashes)):
+ h.logHash(hashes[n])
t = time()-t
print "Add entry (up to %s entries)" %load, timestr(t/float(load))
@@ -454,40 +458,34 @@
print "#================= File ops ====================="
installSignalHandlers(child=1,hup=0,term=0)
dname = mix_mktemp(".d")
- try:
- os.mkdir(dname)
- for i in xrange(200):
- f = open(os.path.join(dname, str(i)), 'wb')
- f.write(s32K)
- f.close()
- lst = [os.path.join(dname,str(i)) for i in range(100) ]
- t1 = time()
- secureDelete(lst)
- t = time()-t1
- print "secureDelete (100x32)", timestr(t)
- waitForChildren()
- t = time()-t1
- print " (sync)", timestr(t)
+ os.mkdir(dname)
+ for i in xrange(200):
+ f = open(os.path.join(dname, str(i)), 'wb')
+ f.write(s32K)
+ f.close()
+ lst = [os.path.join(dname,str(i)) for i in range(100) ]
+ t1 = time()
+ secureDelete(lst)
+ t = time()-t1
+ print "secureDelete (100x32)", timestr(t)
- lst = [ os.path.join(dname,str(i)) for i in range(100,200) ]
- t1 = time()
- for file in lst:
- secureDelete(file)
- t = time()-t1
- print "secureDelete (1)", timestr(t/100)
-
- waitForChildren()
- t = time()-t1
- print " (sync)", timestr(t/100)
+ waitForChildren()
+ t = time()-t1
+ print " (sync)", timestr(t)
+
+ lst = [ os.path.join(dname,str(i)) for i in range(100,200) ]
+ t1 = time()
+ for file in lst:
+ secureDelete(file)
+ t = time()-t1
+ print "secureDelete (1)", timestr(t/100)
+
+ waitForChildren()
+ t = time()-t1
+ print " (sync)", timestr(t/100)
- os.rmdir(dname)
- except:
- for f in os.listdir(dname):
- os.unlink(os.path.join(dname,f))
- os.rmdir(dname)
- raise
#----------------------------------------------------------------------
def testLeaks1():
@@ -548,16 +546,10 @@
#----------------------------------------------------------------------
-def timeAll():
+def timeAll(name, args):
cryptoTiming()
buildMessageTiming()
hashlogTiming()
fileOpsTiming()
serverProcessTiming()
timeEfficiency()
-
-if __name__ == '__main__':
- timeAll()
- #testLeaks1()
- #testLeaks2()
-
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- test.py 21 Aug 2002 19:09:48 -0000 1.21
+++ test.py 25 Aug 2002 03:48:48 -0000 1.22
@@ -23,6 +23,7 @@
import stat
import cPickle
+from mixminion.testSupport import mix_mktemp
from mixminion.Common import MixError, MixFatalError, MixProtocolError, getLog
try:
@@ -30,83 +31,6 @@
except ImportError:
import mixminion._unittest as unittest
-# Test for acceptable permissions and uid on directory?
-_MM_TESTING_TEMPDIR_PARANOIA = 1
-# Holds
-_MM_TESTING_TEMPDIR = None
-_MM_TESTING_TEMPDIR_COUNTER = 0
-_MM_TESTING_TEMPDIR_REMOVE_ON_EXIT = 1
-def mix_mktemp(extra=""):
- '''mktemp wrapper. puts all files under a securely mktemped
- directory.'''
- global _MM_TESTING_TEMPDIR
- global _MM_TESTING_TEMPDIR_COUNTER
- if _MM_TESTING_TEMPDIR is None:
- import tempfile
- temp = tempfile.mktemp()
- paranoia = _MM_TESTING_TEMPDIR_PARANOIA
- if paranoia and os.path.exists(temp):
- print "I think somebody's trying to exploit mktemp."
- sys.exit(1)
- try:
- os.mkdir(temp, 0700)
- except OSError, e:
- print "Something's up with mktemp: %s" % e
- sys.exit(1)
- if not os.path.exists(temp):
- print "Couldn't create temp dir %r" %temp
- sys.exit(1)
- st = os.stat(temp)
- if paranoia:
- if st[stat.ST_MODE] & 077:
- print "Couldn't make temp dir %r with secure permissions" %temp
- sys.exit(1)
- if st[stat.ST_UID] != os.getuid():
- print "The wrong user owns temp dir %r"%temp
- sys.exit(1)
- parent = temp
- while 1:
- p = os.path.split(parent)[0]
- if parent == p:
- break
- parent = p
- st = os.stat(parent)
- m = st[stat.ST_MODE]
- if m & 02 and not (m & stat.S_ISVTX):
- print "Directory %s has fishy permissions %o" %(parent,m)
- sys.exit(1)
- if st[stat.ST_UID] not in (0, os.getuid()):
- print "Directory %s has bad owner %s" % st[stat.UID]
- sys.exit(1)
-
- _MM_TESTING_TEMPDIR = temp
- if _MM_TESTING_TEMPDIR_REMOVE_ON_EXIT:
- import atexit
- atexit.register(deltree, temp)
-
- _MM_TESTING_TEMPDIR_COUNTER += 1
- return os.path.join(_MM_TESTING_TEMPDIR,
- "tmp%05d%s" % (_MM_TESTING_TEMPDIR_COUNTER,extra))
-
-_WAIT_FOR_KIDS = 1
-def deltree(*dirs):
- global _WAIT_FOR_KIDS
- if _WAIT_FOR_KIDS:
- print "Waiting for shred processes to finish."
- waitForChildren()
- _WAIT_FOR_KIDS = 0
- for d in dirs:
- if os.path.isdir(d):
- for fn in os.listdir(d):
- loc = os.path.join(d,fn)
- if os.path.isdir(loc):
- deltree(loc)
- else:
- os.unlink(loc)
- os.rmdir(d)
- elif os.path.exists(d):
- os.unlink(d)
-
def hexread(s):
assert (len(s) % 2) == 0
r = []
@@ -1570,11 +1494,12 @@
class FileParanoiaTests(unittest.TestCase):
def testPrivateDirs(self):
noia = mix_mktemp("noia")
+ tempdir = mixminion.testSupport._MM_TESTING_TEMPDIR
try:
- checkPrivateDir(_MM_TESTING_TEMPDIR)
+ checkPrivateDir(tempdir)
except MixFatalError, e:
self.fail("Can't test directory paranoia, because something's\n"
- +" wrong with %s: %s"%(_MM_TESTING_TEMPDIR,str(e)))
+ +" wrong with %s: %s"%(tempdir, str(e)))
# Nonexistant directory.
self.failUnlessRaises(MixFatalError, checkPrivateDir, noia)
@@ -2311,7 +2236,7 @@
mixminion.Modules.DROP_TYPE, "")
queue = manager.queues['TestModule']
self.failUnless(isinstance(queue,
- mixminion.Modules._SimpleModuleDeliveryQueue))
+ mixminion.Modules.SimpleModuleDeliveryQueue))
self.assertEquals(3, queue.count())
self.assertEquals(exampleMod.processedMessages, [])
try:
@@ -2391,7 +2316,9 @@
suite.addTest(tc(MMTPTests))
return suite
-def testAll():
+def testAll(name, args):
+ init_crypto()
+
# Suppress 'files-can't-be-securely-deleted message while testing'
getLog().setMinSeverity("FATAL")
mixminion.Common.secureDelete([],1)
@@ -2401,7 +2328,3 @@
getLog().setMinSeverity(os.environ.get('MM_TEST_LOGLEVEL', "WARN"))
unittest.TextTestRunner(verbosity=1).run(testSuite())
-
-if __name__ == '__main__':
- init_crypto()
- testAll()