[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Bugfixes for armoring, refactor to use readFile/WriteFi...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv15318/lib/mixminion
Modified Files:
ClientMain.py Common.py Crypto.py Packet.py __init__.py
test.py
Log Message:
Bugfixes for armoring, refactor to use readFile/WriteFile wrappers.
Common:
- Add "tryUnlink" function to perform common unlink-this-file-if-you-can
operation.
- Debug armorText and unarmorText.
- Document writeFile and readFile
Packet:
- Remove dead comment.
test:
- Remove redundant readFile and writeFile
- Add tests for armorText and unarmorText.
ClientMain, Common, ServerInbox, ServerList, EventStats, HashLogs, Modules,
ServerKeys, ServerMain, ServerQueue:
- Use writeFile, readFile, tryUnlink where possible.
- Avoid "except" without "raise" or sys.exc_info()
- Avoid "except OSError" without checking errno or displaying error.
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.86
retrieving revision 1.87
diff -u -d -r1.86 -r1.87
--- ClientMain.py 30 May 2003 03:28:26 -0000 1.86
+++ ClientMain.py 5 Jun 2003 05:24:23 -0000 1.87
@@ -33,15 +33,15 @@
MixError, MixFatalError, MixProtocolError, MixProtocolBadAuth, UIError, \
UsageError, ceilDiv, createPrivateDir, isPrintingAscii, isSMTPMailbox, \
formatDate, formatFnameTime, formatTime, Lockfile, openUnique, \
- previousMidnight, readPickled, readPossiblyGzippedFile, secureDelete, \
- stringContains, succeedingMidnight, writePickled
+ previousMidnight, readFile, readPickled, readPossiblyGzippedFile, \
+ secureDelete, stringContains, succeedingMidnight, tryUnlink, writeFile, \
+ writePickled
from mixminion.Crypto import sha1, ctr_crypt, trng
from mixminion.Config import ClientConfig, ConfigError
from mixminion.ServerInfo import ServerInfo, ServerDirectory
from mixminion.Packet import ParseError, parseMBOXInfo, parseReplyBlocks, \
- parseSMTPInfo, parseTextEncodedMessages, parseTextReplyBlocks,\
- ReplyBlock,\
- MBOX_TYPE, SMTP_TYPE, DROP_TYPE
+ parseSMTPInfo, parseTextEncodedMessages, parseTextReplyBlocks, \
+ ReplyBlock, MBOX_TYPE, SMTP_TYPE, DROP_TYPE
# FFFF This should be made configurable and adjustable.
MIXMINION_DIRECTORY_URL = "http://mixminion.net/directory/Directory.gz";
@@ -207,11 +207,7 @@
if fp and mixminion.Crypto.pk_fingerprint(identity) != fp:
raise MixFatalError("Bad identity key on directory")
- try:
- os.unlink(os.path.join(self.dir, "cache"))
- except OSError, e:
- if e.errno != errno.ENOENT:
- raise
+ tryUnlink(os.path.join(self.dir, "cache"))
# Install the new directory
if gz:
@@ -702,7 +698,7 @@
for a in allowed:
try:
t = mixminion.parse_version_string(a)
- except:
+ except ValueError:
LOG.warn("Couldn't parse recommended version %s", a)
continue
try:
@@ -987,10 +983,7 @@
def _checkMagic(self, fn, magic):
"""Make sure that the magic string on a given key file %s starts with
is equal to 'magic'. Raise MixError if it isn't."""
- f = open(fn, 'rb')
- s = f.read()
- f.close()
- if not s.startswith(magic):
+ if not readFile(fn, 1).startswith(magic):
raise MixError("Invalid versioning on key file")
def _save(self, fn, data, magic, password):
@@ -1007,11 +1000,10 @@
def _load(self, fn, magic, password):
"""Load and return the key stored in 'fn' using the magic string
'magic' and the password 'password'. Raise MixError on failure."""
- f = open(fn, 'rb')
- s = f.read()
- f.close()
+ s = readFile(fn, 1)
if not s.startswith(magic):
- raise MixError("Invalid key file")
+ raise MixError("Invalid versioning on key file")
+
s = s[len(magic):]
if len(s) < 8:
raise MixError("Key file too short")
@@ -1068,8 +1060,9 @@
"""Create a default, 'fail-safe' configuration in a given file"""
LOG.warn("No configuration file found. Installing default file in %s",
fname)
- f = open(os.path.expanduser(fname), 'w')
- f.write("""\
+
+ writeFile(os.path.expanduser(fname),
+ """\
# This file contains your options for the mixminion client.
[Host]
## Use this option to specify a 'secure remove' command.
@@ -1107,7 +1100,7 @@
ConnectionTimeout: 20 seconds
""")
- f.close()
+
class SURBLog:
"""A SURBLog manipulates a database on disk to remember which SURBs we've
@@ -1964,11 +1957,10 @@
if fn == '-':
s = sys.stdin.read()
else:
- f = open(fn, 'rb')
- s = f.read()
- f.close()
+ s = readFile(fn, 1)
try:
- if stringContains(s, "== BEGIN TYPE III REPLY BLOCK =="):
+ if stringContains(s,
+ "-----BEGIN TYPE III REPLY BLOCK-----"):
surbs.extend(parseTextReplyBlocks(s))
else:
surbs.extend(parseReplyBlocks(s))
@@ -2175,15 +2167,12 @@
if inFile is None:
inFile = "-"
- if inFile == '-':
- f = sys.stdin
- print "Enter your message now. Type Ctrl-D when you are done."
- else:
- f = open(inFile, 'r')
-
try:
- payload = f.read()
- f.close()
+ if inFile == '-':
+ print "Enter your message now. Type Ctrl-D when you are done."
+ payload = sys.stdin.read()
+ else:
+ payload = readFile(inFile)
except KeyboardInterrupt:
print "Interrupted. Message not sent."
sys.exit(1)
@@ -2423,9 +2412,7 @@
s = sys.stdin.read()
else:
try:
- f = open(inputFile, 'r')
- s = f.read()
- f.close()
+ s = readFile(inputFile)
except OSError, e:
LOG.error("Could not read file %s: %s", inputFile, e)
try:
@@ -2578,12 +2565,10 @@
try:
for fn in args:
- f = open(fn, 'rb')
- s = f.read()
- f.close()
+ s = readFile(fn, 1)
print "==== %s"%fn
try:
- if stringContains(s, "== BEGIN TYPE III REPLY BLOCK =="):
+ if stringContains(s, "-----BEGIN TYPE III REPLY BLOCK-----"):
surbs = parseTextReplyBlocks(s)
else:
surbs = parseReplyBlocks(s)
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.85
retrieving revision 1.86
diff -u -d -r1.85 -r1.86
--- Common.py 30 May 2003 03:11:46 -0000 1.85
+++ Common.py 5 Jun 2003 05:24:23 -0000 1.86
@@ -13,8 +13,8 @@
'installSIGCHLDHandler', 'isSMTPMailbox', 'openUnique',
'previousMidnight', 'readFile', 'readPickled',
'readPossiblyGzippedFile', 'secureDelete', 'stringContains',
- 'succeedingMidnight', 'unarmorText', 'waitForChildren',
- 'writeFile', 'writePickled' ]
+ 'succeedingMidnight', 'tryUnlink', 'unarmorText',
+ 'waitForChildren', 'writeFile', 'writePickled' ]
import binascii
import bisect
@@ -172,7 +172,6 @@
headers 'header'.
If base64 is false, uses cleartext armor."""
- #XXXX004 testme
result = []
result.append("-----BEGIN %s-----\n" %type)
for k,v in headers:
@@ -189,7 +188,7 @@
return "".join(result)
# Matches a begin line.
-BEGIN_LINE_RE = re.compile(r'^-----BEGIN ([^-]+)-----\s*$',re.M)
+BEGIN_LINE_RE = re.compile(r'^-----BEGIN ([^-]+)-----[ \t]*$',re.M)
# Matches a header line.
ARMOR_KV_RE = re.compile(r'([^:\s]+): ([^\n]+)')
@@ -204,7 +203,6 @@
base64fn -- if provided, called with (type, headers) to tell whether
we do cleartext armor.
"""
- #XXXX004 testme
result = []
while 1:
@@ -217,7 +215,7 @@
return result
tp = mBegin.group(1)
- endPat = r"^-----END %s-----$" % tp
+ endPat = r"^-----END %s-----[ \t]*$" % tp
endRE = re.compile(endPat, re.M)
mEnd = endRE.search(s, mBegin.start())
@@ -225,15 +223,17 @@
raise ValueError("Couldn't find end line for '%s'"%tp.lower())
if tp not in findTypes:
- idx = mEnd.end+1
+ s = s[mEnd.end()+1:]
continue
-
+
idx = mBegin.end()+1
endIdx = mEnd.start()
+
assert s[idx-1] == s[endIdx-1] == '\n'
while idx < endIdx:
nl = s.index("\n", idx, endIdx)
line = s[idx:nl]
+ idx = nl+1
if ":" in line:
m = ARMOR_KV_RE.match(line)
if not m:
@@ -241,12 +241,10 @@
fields.append((m.group(1), m.group(2)))
elif line.strip() == '':
break
- idx = nl+1
if base64fn:
base64 = base64fn(tp,fields)
- idx = nl+1
if base64:
try:
value = binascii.a2b_base64(s[idx:endIdx])
@@ -419,8 +417,7 @@
LOG.error("Atomic file not closed/discarded: %s",self.tmpname)
def readFile(fn, binary=0):
- """DOCDOC"""
- #XXXX004 use more.
+ """Return the contents of the file named <fn>."""
f = open(fn, ['r', 'rb'][binary])
try:
return f.read()
@@ -428,8 +425,14 @@
f.close()
def writeFile(fn, contents, mode=0600, binary=0):
- """DOCDOC"""
- #XXXX004 use more.
+ """Atomically write a string <contents> into a file <file> with mode
+ <mode>. If <binary>, binary mode will be used.
+
+ If the file exists, it will be replaced.
+
+ If two processes attempt to writeFile the same file at once,
+ the one finishing last wins.
+ """
tmpname = fn+".tmp"
f, tmpname = openUnique(tmpname, ['w','wb'][binary], mode)
try:
@@ -469,6 +472,18 @@
os.rename(tmpname, fn)
+def tryUnlink(fname):
+ """Try to remove the file named fname. If the file is erased, return 1.
+ If the file didn't exist in the first place, return 0. Otherwise
+ propagate an exception."""
+ try:
+ os.unlink(fname)
+ return 1
+ except OSError, e:
+ if e.errno == errno.ENOENT:
+ return 0
+ raise
+
#----------------------------------------------------------------------
# Secure filesystem operations.
@@ -621,9 +636,9 @@
if not os.path.exists(parent):
createPrivateDir(parent)
self.file = open(self.fname, 'a')
- except OSError:
+ except OSError, e:
self.file = None
- raise MixError("Unable to open log file %r"%self.fname)
+ raise MixError("Unable to open log file %r: %s"%self.fname, e)
def close(self):
"Close the underlying file"
self.file.close()
@@ -1250,11 +1265,7 @@
def getContents(self):
"""Return the contents of the lock file."""
- try:
- f = open(self.filename, 'r')
- return f.read()
- finally:
- f.close()
+ return readFile(self.filename)
def acquire(self, contents="", blocking=0):
"""Acquire this lock. If we're acquiring the lock for the first time,
Index: Crypto.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Crypto.py,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -d -r1.43 -r1.44
--- Crypto.py 17 May 2003 00:08:42 -0000 1.43
+++ Crypto.py 5 Jun 2003 05:24:23 -0000 1.44
@@ -51,7 +51,8 @@
except MixFatalError:
raise
except:
- raise MixFatalError("Error initializing entropy source")
+ info = sys.exc_info()
+ raise MixFatalError("Error initializing entropy source: %s", info[0])
openssl_seed(40)
def sha1(s):
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- Packet.py 28 May 2003 06:37:35 -0000 1.44
+++ Packet.py 5 Jun 2003 05:24:23 -0000 1.45
@@ -361,24 +361,6 @@
return a list containing the reply blocks. Raise ParseError on
failure."""
-## while 1:
-## m = RB_TEXT_RE.search(s[idx:])
-## if m is None:
-## # FFFF Better errors on malformatted reply blocks.
-## break
-## version, text = m.group(1), m.group(2)
-## idx += m.end()
-## if version != '0.1':
-## LOG.warn("Skipping reply block with unrecognized version: %s",
-## version)
-## continue
-## try:
-## val = binascii.a2b_base64(text)
-## except (TypeError, binascii.Incomplete, binascii.Error), e:
-## raise ParseError("Bad reply block encoding: %s"%e)
-## blocks.append(parseReplyBlock(val))
-## return blocks
-
try:
res = unarmorText(s, (RB_ARMOR_NAME,), base64=1)
except ValueError, e:
Index: __init__.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/__init__.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- __init__.py 2 Jun 2003 17:49:51 -0000 1.36
+++ __init__.py 5 Jun 2003 05:24:23 -0000 1.37
@@ -77,7 +77,7 @@
try:
patch = int(patch)
except ValueError:
- pass
+ patch = patch
return (int(major), int(minor), int(sub), status, patch)
def cmp_versions(a,b):
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.115
retrieving revision 1.116
diff -u -d -r1.115 -r1.116
--- test.py 3 Jun 2003 17:28:11 -0000 1.115
+++ test.py 5 Jun 2003 05:24:23 -0000 1.116
@@ -107,24 +107,6 @@
else:
return abs(f1-f2) < .00001
-def readFile(fname):
- """Return the contents of the file named 'fname'. We could just say
- 'open(fname).read()' instead, but that isn't as clean."""
- f = open(fname, 'r')
- try:
- return f.read()
- finally:
- f.close()
-
-def writeFile(fname, contents):
- """Create a new file named fname, replacing any such file that exists,
- with the contents 'contents'."""
- f = open(fname, 'w')
- try:
- f.write(contents)
- finally:
- f.close()
-
#----------------------------------------------------------------------
# RSA key caching functionality
@@ -454,6 +436,20 @@
t(36 not in fromSquareToSquare)
t(100 not in fromSquareToSquare)
+ def _intervalEq(self, a, *others):
+ eq = self.assertEquals
+ for b in others:
+ if isinstance(b, IntervalSet):
+ eq(a,b)
+ b._checkRep()
+ elif isinstance(b, types.StringType):
+ eq(repr(a), b)
+ elif isinstance(b, types.ListType):
+ eq(a.getIntervals(), b)
+ else:
+ raise MixError()
+ a._checkRep()
+
def test_openUnique(self):
d = mix_mktemp()
os.mkdir(d)
@@ -524,20 +520,94 @@
self.assertEquals(len(line), max)
self.assert_(len(lines[-1]) <= max)
- def _intervalEq(self, a, *others):
- eq = self.assertEquals
- for b in others:
- if isinstance(b, IntervalSet):
- eq(a,b)
- b._checkRep()
- elif isinstance(b, types.StringType):
- eq(repr(a), b)
- elif isinstance(b, types.ListType):
- eq(a.getIntervals(), b)
- else:
- raise MixError()
- a._checkRep()
+ def test_armor(self):
+ inp1 = "xyzzy111"*10
+ inp2 = "Hello\nWorld!\n--Me.\n"
+ # Simple base64
+ self.assertEquals(armorText(inp1, "FOO BAR", (), 1),
+"""-----BEGIN FOO BAR-----
+
+eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTF4eXp6eTExMXh5enp5MTEx
+eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTE=
+-----END FOO BAR-----\n""")
+ # With headers
+ self.assertEquals(armorText(inp1, "FOO BAR",
+ [("H1", "xyz"),("H2", "a b")], 1),
+"""-----BEGIN FOO BAR-----
+H1: xyz
+H2: a b
+
+eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTF4eXp6eTExMXh5enp5MTEx
+eHl6enkxMTF4eXp6eTExMXh5enp5MTExeHl6enkxMTE=
+-----END FOO BAR-----\n""")
+
+ # No base64
+ self.assertEquals(armorText(inp1, "FOO BAR", (), 0),
+"""-----BEGIN FOO BAR-----
+
+xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111xyzzy111
+-----END FOO BAR-----\n""")
+
+ # No base64, hyphen escape.
+ self.assertEquals(armorText(inp2, "FOO BAR", [("H3", "text")], 0),
+"""-----BEGIN FOO BAR-----
+H3: text
+
+Hello
+World!
+- --Me.
+-----END FOO BAR-----\n""")
+
+ # Test unarmor, succeeding cases.
+ for inp in inp1, inp2:
+ for mode in 0,1:
+ for headers in [], [("H-x", "abcde fg")]:
+ enc = armorText(inp, "THIS THAT", headers, mode)
+ enc = "asdasdkjlasd\nHey kids!\n"+enc
+ tp, h, b = unarmorText(enc, ["THIS THAT", "FRED"],
+ mode)[0]
+ self.assertEquals(tp, "THIS THAT")
+ self.assertEquals(h, headers)
+ self.assertEquals(b.strip(), inp.strip())
+ if not mode:
+ self.assert_(b.endswith("\n"))
+ # Test base64fn and concatenation.
+ enc1 = armorText(inp2, "THIS THAT", [("H-64", "0")], 0)
+ enc2 = armorText(inp2, "THIS THAT", [("H-64", "1")], 1)
+ def base64fn(tp, h, assert_=self.assert_):
+ assert_(tp=="THIS THAT")
+ assert_(len(h) == 1)
+ assert_(len(h[0]) == 2)
+ assert_(h[0][0] == "H-64")
+ return int(h[0][1])
+ dec = unarmorText(enc1+enc2, ["THIS THAT"], base64fn=base64fn)
+ self.assertEquals(len(dec), 2)
+ for mode in 0, 1:
+ tp,h,b = dec[mode]
+ self.assertEquals(tp, "THIS THAT")
+ self.assertEquals(h, [("H-64", str(mode))])
+ self.assertEquals(b, inp2)
+
+ # Test skipping unwanted types
+ enc3 = armorText(inp2, "OTHER", [("H-64", "1")], 1)
+ dec2 = unarmorText(enc1+enc3+enc2, ["THIS THAT"], base64fn=base64fn)
+ self.assertEquals(dec2, dec)
+
+ # Test skipping broken armor.
+ self.assertRaises(ValueError,
+ unarmorText,"-----BEGIN X-----\n\n", ["Y"], 1)
+ self.assertRaises(ValueError,
+ unarmorText,
+ "-----BEGIN X-----\n"
+ ":B:C\n\n"
+ "A B C\n-----END X-----\n", ["X"], 0)
+ self.assertRaises(ValueError,
+ unarmorText,
+ "-----BEGIN X-----\n"
+ "A: X\n\n"
+ "A B C\n-----END X-----\n", ["X"], 1)
+
#----------------------------------------------------------------------
class MinionlibCryptoTests(unittest.TestCase):
@@ -5746,7 +5816,7 @@
tc = loader.loadTestsFromTestCase
if 0:
- suite.addTest(tc(MMTPTests))
+ suite.addTest(tc(MiscTests))
return suite
suite.addTest(tc(MiscTests))