[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Fix bugs in text armor handling; switch to OpenPGP armor
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv15299/lib/mixminion
Modified Files:
ClientMain.py Common.py Packet.py test.py
Log Message:
Fix bugs in text armor handling; switch to OpenPGP armor
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.73
retrieving revision 1.74
diff -u -d -r1.73 -r1.74
--- ClientMain.py 17 May 2003 00:08:41 -0000 1.73
+++ ClientMain.py 21 May 2003 18:03:33 -0000 1.74
@@ -37,7 +37,8 @@
from mixminion.Config import ClientConfig, ConfigError
from mixminion.ServerInfo import ServerInfo, ServerDirectory
from mixminion.Packet import ParseError, parseMBOXInfo, parseReplyBlocks, \
- parseSMTPInfo, parseTextEncodedMessage, parseTextReplyBlocks, ReplyBlock,\
+ parseSMTPInfo, parseTextEncodedMessages, parseTextReplyBlocks,\
+ ReplyBlock,\
MBOX_TYPE, SMTP_TYPE, DROP_TYPE
# FFFF This should be made configurable and adjustable.
@@ -1565,10 +1566,7 @@
#XXXX004 write unit tests
results = []
idx = 0
- while idx < len(s):
- msg, idx = parseTextEncodedMessage(s, idx=idx, force=force)
- if msg is None:
- return results
+ for msg in parseTextEncodedMessages(s, force=force):
if msg.isOvercompressed() and not force:
LOG.warn("Message is a possible zlib bomb; not uncompressing")
if not msg.isEncrypted():
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.75
retrieving revision 1.76
diff -u -d -r1.75 -r1.76
--- Common.py 17 May 2003 05:40:42 -0000 1.75
+++ Common.py 21 May 2003 18:03:33 -0000 1.76
@@ -7,12 +7,14 @@
__all__ = [ 'IntervalSet', 'Lockfile', 'LOG', 'LogStream', 'MixError',
'MixFatalError', 'MixProtocolError', 'UIError', 'UsageError',
+ 'armorText',
'ceilDiv', 'checkPrivateDir', 'checkPrivateFile',
'createPrivateDir', 'encodeBase64', 'floorDiv', 'formatBase64',
'formatDate', 'formatFnameTime', 'formatTime',
'installSIGCHLDHandler', 'isSMTPMailbox', 'openUnique',
'previousMidnight', 'readPickled', 'readPossiblyGzippedFile',
'secureDelete', 'stringContains', 'succeedingMidnight',
+ 'unarmorText',
'waitForChildren', 'writePickled' ]
import binascii
@@ -155,6 +157,110 @@
return "".join([ s.strip() for s in pieces ])
else:
return "".join(pieces)
+
+#----------------------------------------------------------------------
+# Functions to generate and parse OpenPGP-style ASCII armor
+
+# Matches a line that needs to be ascii-armored in plaintext mode.
+DASH_ARMOR_RE = re.compile('^-', re.M)
+
+def armorText(s, type, headers=(), base64=1):
+ """Given a string (s), string holding a message type (type), and a
+ list of key-value pairs for headers, generates an OpenPGP-style
+ ASCII-armored message of type 'type', with contents 's' and
+ headers 'header'.
+
+ If base64 is false, uses cleartext armor."""
+ #XXXX004 testme
+ result = []
+ result.append("-----BEGIN %s-----\n" %type)
+ for k,v in headers:
+ result.append("%s: %s\n" %(k,v))
+ result.append("\n")
+ if base64:
+ result.append(encodeBase64(s, lineWidth=64))
+ else:
+ result.append(DASH_ARMOR_RE.sub('- -', s))
+ if not result[-1].endswith("\n"):
+ result.append("\n")
+ result.append("-----END %s-----\n" %type)
+
+ return "".join(result)
+
+# Matches a begin line.
+BEGIN_LINE_RE = re.compile(r'^-----BEGIN ([^-]+)-----\s*$',re.M)
+
+# Matches a header line.
+ARMOR_KV_RE = re.compile(r'([^:\s]+): ([^\n]+)')
+def unarmorText(s, findTypes, base64=1, base64fn=None):
+ """Parse a list of OpenPGP-style ASCII-armored messages from 's',
+ and return a list of (type, headers, body) tuples, where 'headers'
+ is a list of key,val tuples.
+
+ s -- the string to parse.
+ findTypes -- a list of types to search for; others are ignored.
+ base64 -- if false, we do cleartext armor.
+ base64fn -- if provided, called with (type, headers) to tell whether
+ we do cleartext armor.
+ """
+ #XXXX004 testme
+ result = []
+
+ while 1:
+ tp = None
+ fields = []
+ value = None
+
+ mBegin = BEGIN_LINE_RE.search(s)
+ if not mBegin:
+ return result
+
+ tp = mBegin.group(1)
+ endPat = r"^-----END %s-----$" % tp
+
+ endRE = re.compile(endPat, re.M)
+ mEnd = endRE.search(s, mBegin.start())
+ if not mEnd:
+ raise ValueError("Couldn't find end line for '%s'"%tp.lower())
+
+ if tp not in findTypes:
+ idx = mEnd.end+1
+ continue
+
+ idx = mBegin.end()+1
+ endIdx = mEnd.start()
+ assert s[idx-1] == s[endIdx-1] == '\n'
+ while idx < endIdx:
+ nl = s.index("\n", idx, endIdx)
+ line = s[idx:nl]
+ if ":" in line:
+ m = ARMOR_KV_RE.match(line)
+ if not m:
+ raise ValueError("Bad header for '%s'"%tp.lower())
+ fields.append((m.group(1), m.group(2)))
+ elif line.strip() == '':
+ break
+ idx = nl+1
+
+ if base64fn:
+ base64 = base64fn(tp,fields)
+
+ idx = nl+1
+ if base64:
+ try:
+ value = binascii.a2b_base64(s[idx:endIdx])
+ except (TypeError, binascii.Incomplete, binascii.Error), e:
+ raise ValueError(str(e))
+ else:
+ v = s[idx:endIdx].split("\n")
+ for i in xrange(len(v)):
+ if v[i].startswith("- "):
+ v[i] = v[i][2:]
+ value = "\n".join(v)
+
+ result.append((tp, fields, value))
+
+ s = s[mEnd.end()+1:]
#----------------------------------------------------------------------
def checkPrivateFile(fn, fix=1):
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- Packet.py 17 May 2003 00:08:43 -0000 1.41
+++ Packet.py 21 May 2003 18:03:33 -0000 1.42
@@ -22,8 +22,7 @@
'parseHeader', 'parseIPV4Info',
'parseMBOXInfo', 'parseMessage', 'parsePayload', 'parseReplyBlock',
'parseReplyBlocks', 'parseSMTPInfo', 'parseSubheader',
- 'parseTextEncodedMessage', 'parseTextReplyBlocks', 'uncompressData'
- ]
+ 'parseTextEncodedMessages', 'parseTextReplyBlocks', 'uncompressData' ]
import binascii
import re
@@ -32,7 +31,7 @@
import zlib
from socket import inet_ntoa, inet_aton
from mixminion.Common import MixError, MixFatalError, encodeBase64, \
- floorDiv, formatTime, isSMTPMailbox, LOG
+ floorDiv, formatTime, isSMTPMailbox, LOG, armorText, unarmorText
from mixminion.Crypto import sha1
if sys.version_info[:3] < (2,2,0):
@@ -355,35 +354,43 @@
# routingInfo for the last server.
RB_UNPACK_PATTERN = "!4sBBL%dsHH%ss" % (HEADER_LEN, SECRET_LEN)
MIN_RB_LEN = 30+HEADER_LEN
-RB_TEXT_START = "======= BEGIN TYPE III REPLY BLOCK ======="
-RB_TEXT_END = "======== END TYPE III REPLY BLOCK ========"
-# XXXX Use a better pattern here.
-RB_TEXT_RE = re.compile(r"==+ BEGIN TYPE III REPLY BLOCK ==+"+
- r'[\r\n]+Version: (\d+\.\d+)\s*[\r\n]+(.*?)'+
- r"==+ END TYPE III REPLY BLOCK ==+", re.M|re.DOTALL)
+RB_ARMOR_NAME = "TYPE III REPLY BLOCK"
def parseTextReplyBlocks(s):
"""Given a string holding one or more text-encoded reply blocks,
return a list containing the reply blocks. Raise ParseError on
failure."""
- idx = 0
+
+## while 1:
+## m = RB_TEXT_RE.search(s[idx:])
+## if m is None:
+## # FFFF Better errors on malformatted reply blocks.
+## break
+## version, text = m.group(1), m.group(2)
+## idx += m.end()
+## if version != '0.1':
+## LOG.warn("Skipping reply block with unrecognized version: %s",
+## version)
+## continue
+## try:
+## val = binascii.a2b_base64(text)
+## except (TypeError, binascii.Incomplete, binascii.Error), e:
+## raise ParseError("Bad reply block encoding: %s"%e)
+## blocks.append(parseReplyBlock(val))
+## return blocks
+
+ try:
+ res = unarmorText(s, (RB_ARMOR_NAME,), base64=1)
+ except ValueError, e:
+ raise ParseError(str(e))
blocks = []
- while 1:
- m = RB_TEXT_RE.search(s[idx:])
- if m is None:
- # FFFF Better errors on malformatted reply blocks.
- break
- version, text = m.group(1), m.group(2)
- idx += m.end()
- if version != '0.1':
- LOG.warn("Skipping reply block with unrecognized version: %s",
- version)
- continue
- try:
- val = binascii.a2b_base64(text)
- except (TypeError, binascii.Incomplete, binascii.Error), e:
- raise ParseError("Bad reply block encoding: %s"%e)
- blocks.append(parseReplyBlock(val))
+ for tp, fields, value in res:
+ d = {}
+ for k,v in fields:
+ d[k]=v
+ if not d.get("Version") == '0.2':
+ LOG.warn("Skipping SURB with bad version: %r", d.get("Version"))
+ blocks.append(parseReplyBlock(value))
return blocks
def parseReplyBlocks(s):
@@ -472,10 +479,8 @@
def packAsText(self):
"""Returns the external text representation of this reply block"""
- text = encodeBase64(self.pack())
- if not text.endswith("\n"):
- text += "\n"
- return "%s\nVersion: 0.1\n\n%s%s\n"%(RB_TEXT_START,text,RB_TEXT_END)
+ return armorText(self.pack(), RB_ARMOR_NAME,
+ headers=(("Version", "0.2"),))
#----------------------------------------------------------------------
# Routing info
@@ -575,83 +580,60 @@
# The format is HeaderLine, TagLine?, Body, FooterLine.
# TagLine is one of /Message-type: (overcompressed|binary)/
# or /Decoding-handle: (base64-encoded-stuff)/.
-MESSAGE_START_LINE = "======= TYPE III ANONYMOUS MESSAGE BEGINS ======="
-MESSAGE_END_LINE = "======== TYPE III ANONYMOUS MESSAGE ENDS ========"
-_MESSAGE_START_RE = re.compile(r"==+ TYPE III ANONYMOUS MESSAGE BEGINS ==+")
-_MESSAGE_END_RE = re.compile(r"==+ TYPE III ANONYMOUS MESSAGE ENDS ==+")
-_FIRST_LINE_RE = re.compile(r'''^Decoding-handle:\s(.*)\r*\n|
- Message-type:\s(.*)\r*\n''', re.X+re.S)
-_LINE_RE = re.compile(r'[^\r\n]*\r*\n', re.S+re.M)
-
-def _nextLine(s, idx):
- """Helper method. Return the index of the first character of the first
- line of s to follow <idx>."""
- m = _LINE_RE.match(s[idx:])
- if m is None:
- return len(s)
- else:
- return m.end()+idx
+#DOCDOC
-def parseTextEncodedMessage(msg,force=0,idx=0):
+MESSAGE_ARMOR_NAME = "TYPE III ANONYMOUS MESSAGE"
+
+def parseTextEncodedMessages(msg,force=0):
"""Given a text-encoded Type III packet, return a TextEncodedMessage
- object or raise ParseError.
+ object or raise ParseError. DOCDOC
force -- uncompress the message even if it's overcompressed.
idx -- index within <msg> to search.
"""
- #idx = msg.find(MESSAGE_START_PAT, idx)
- m = _MESSAGE_START_RE.search(msg[idx:])
- if m is None:
- return None, None
- idx += m.start()
- m = _MESSAGE_END_RE.search(msg[idx:])
- if m is None:
- raise ParseError("No end line found")
- msgEndIdx = idx+m.start()
- idx = _nextLine(msg, idx)
- firstLine = msg[idx:_nextLine(msg, idx)]
- m = _FIRST_LINE_RE.match(firstLine)
- if m is None:
- msgType = 'TXT'
- elif m.group(1):
- ascTag = m.group(1)
- msgType = "ENC"
- idx = _nextLine(msg, idx)
- elif m.group(2):
- if m.group(2) == 'overcompressed':
- msgType = 'LONG'
- elif m.group(2) == 'binary':
- msgType = 'BIN'
+ def isBase64(t,f):
+ for k,v in f:
+ if k == "Message-type":
+ if v != 'plaintext':
+ return 1
+ return 0
+
+ unarmored = unarmorText(msg, (MESSAGE_ARMOR_NAME,), base64fn=isBase64)
+ res = []
+ for tp,fields,val in unarmored:
+ d = {}
+ for k,v in fields:
+ d[k] = v
+ if d.get("Message-type", "plaintext") == "plaintext":
+ msgType = 'TXT'
+ elif d['Message-type'] == 'overcompressed':
+ msgType = "LONG"
+ elif d['Message-type'] == 'binary':
+ msgType = "BIN"
+ elif d['Message-type'] == 'encrypted':
+ msgType = "ENC"
else:
raise ParseError("Unknown message type: %r"%m.group(2))
- idx = _nextLine(msg, idx)
-
- endIdx = _nextLine(msg, msgEndIdx)
- msg = msg[idx:msgEndIdx]
- if msgType == 'TXT':
- return TextEncodedMessage(msg, 'TXT'), endIdx
+ ascTag = d.get("Decoding-handle")
+ if ascTag:
+ msgType = "ENC"
- try:
- msg = binascii.a2b_base64(msg)
- except (TypeError, binascii.Incomplete, binascii.Error), e:
- raise ParseError("Error in base64 encoding: %s"%e)
+ if msgType == 'LONG' and force:
+ msg = uncompressData(msg)
+
+ if msgType in ('TXT','BIN','LONG'):
+ res.append(TextEncodedMessage(val, msgType))
+ else:
+ assert msgType == 'ENC'
+ try:
+ tag = binascii.a2b_base64(ascTag)
+ except (TypeError, binascii.Incomplete, binascii.Error), e:
+ raise ParseError("Error in base64 encoding: %s"%e)
+ if len(tag) != TAG_LEN:
+ raise ParseError("Impossible tag length: %s"%len(tag))
+ res.append(TextEncodedMessage(val, 'ENC', tag))
- if msgType == 'BIN':
- return TextEncodedMessage(msg, 'BIN'), endIdx
- elif msgType == 'LONG':
- if force:
- msg = uncompressData(msg) # May raise ParseError
- return TextEncodedMessage(msg, 'LONG'), endIdx
- elif msgType == 'ENC':
- try:
- tag = binascii.a2b_base64(ascTag)
- except (TypeError, binascii.Incomplete, binascii.Error), e:
- raise ParseError("Error in base64 encoding: %s"%e)
- if len(tag) != TAG_LEN:
- raise ParseError("Impossible tag length: %s"%len(tag))
- return TextEncodedMessage(msg, 'ENC', tag), endIdx
- else:
- raise MixFatalError("unreached")
+ return res
class TextEncodedMessage:
"""A TextEncodedMessage object holds a Type III message as delivered
@@ -686,30 +668,18 @@
def pack(self):
"""Return the text representation of this message."""
c = self.contents
- preNL = postNL = ""
-
- if self.messageType != 'TXT':
- c = encodeBase64(c)
- else:
- if (c.startswith("Decoding-handle:") or
- c.startswith("Message-type:")):
- preNL = "\n"
-
- if self.messageType == 'TXT':
- tagLine = ""
- elif self.messageType == 'ENC':
- ascTag = binascii.b2a_base64(self.tag).strip()
- tagLine = "Decoding-handle: %s\n\n" % ascTag
- elif self.messageType == 'LONG':
- tagLine = "Message-type: overcompressed\n\n"
- elif self.messageType == 'BIN':
- tagLine = "Message-type: binary\n\n"
-
- if c and c[-1] != '\n':
- postNL = "\n"
+ fields = [("Message-type",
+ { 'TXT' : "plaintext",
+ 'LONG' : "overcompressed",
+ 'BIN' : "binary",
+ 'ENC' : "encrypted" }[self.messageType]),
+ ]
+ if self.messageType == 'ENC':
+ fields.append(("Decoding-handle",
+ binascii.b2a_base64(self.tag).strip()))
- return "%s\n%s%s%s%s%s\n" % (
- MESSAGE_START_LINE, tagLine, preNL, c, postNL, MESSAGE_END_LINE)
+ return armorText(c, MESSAGE_ARMOR_NAME, headers=fields,
+ base64=(self.messageType!='TXT'))
#----------------------------------------------------------------------
# COMPRESSION FOR PAYLOADS
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.104
retrieving revision 1.105
diff -u -d -r1.104 -r1.105
--- test.py 17 May 2003 00:08:43 -0000 1.104
+++ test.py 21 May 2003 18:03:33 -0000 1.105
@@ -1201,18 +1201,19 @@
def testTextEncodedMessage(self):
tem = TextEncodedMessage
- ptem = parseTextEncodedMessage
+ ptem = parseTextEncodedMessages
eq = self.assertEquals
- start = "======= TYPE III ANONYMOUS MESSAGE BEGINS =======\n"
- end = "======== TYPE III ANONYMOUS MESSAGE ENDS ========\n"
+ start = "-----BEGIN TYPE III ANONYMOUS MESSAGE-----\n"
+ pt = "Message-type: plaintext\n"
+ end = "-----END TYPE III ANONYMOUS MESSAGE-----\n"
# Test generation: text case
mt1 = tem("Hello, whirled","TXT")
- eq(mt1.pack(), start+"Hello, whirled\n"+end)
+ eq(mt1.pack(), start+pt+"\nHello, whirled\n"+end)
mt2 = tem("Hello, whirled\n", "TXT")
- eq(mt2.pack(), start+"Hello, whirled\n"+end)
+ eq(mt2.pack(), start+pt+"\nHello, whirled\n"+end)
mt3 = tem("Decoding-handle: gotcha!\nFoobar\n", "TXT")
- eq(mt3.pack(), start+"\nDecoding-handle: gotcha!\nFoobar\n"+end)
+ eq(mt3.pack(), start+pt+"\nDecoding-handle: gotcha!\nFoobar\n"+end)
# Text generation: binary case
v = hexread("00D1E50FED1F1CE5")*12
v64 = encodeBase64(v)
@@ -1230,21 +1231,21 @@
# Encoded
menc1 = tem(v, "ENC", "9"*20)
tag64 = base64.encodestring("9"*20).strip()
- eq(menc1.pack(), start+"Decoding-handle: "+tag64+"\n\n"+v64+end)
+ eq(menc1.pack(), start+
+ "Message-type: encrypted\nDecoding-handle: "+tag64+"\n\n"+v64+end)
# Test parsing: successful cases
p = ptem(mt1.pack())[0]
eq(p.pack(), mt1.pack())
eq(p.getContents(), "Hello, whirled\n")
self.assert_(p.isText())
- p = ptem("This message is a test of the emergent broadcast system?\n "
+ p = ptem("This message is a test of the emergent broadcast system?\n"
+mt2.pack())[0]
eq(p.pack(), mt2.pack())
eq(p.getContents(), "Hello, whirled\n")
# Two concatenated message.
s = mb1.pack() + "\n\n" + ml1.pack()
- p, i = ptem(s)
- p2, _ = ptem(s, idx=i)
+ p, p2 = ptem(s)
eq(p.pack(), mb1.pack())
self.assert_(p.isBinary())
eq(p.getContents(), v)
@@ -1878,9 +1879,9 @@
self.assertEquals(reply.pack(), parseReplyBlock(reply.pack()).pack())
txt = reply.packAsText()
self.assert_(txt.startswith(
- "======= BEGIN TYPE III REPLY BLOCK =======\nVersion: 0.1\n"))
+ "-----BEGIN TYPE III REPLY BLOCK-----\nVersion: 0.2\n\n"))
self.assert_(txt.endswith(
- "\n======== END TYPE III REPLY BLOCK ========\n"))
+ "-----END TYPE III REPLY BLOCK-----\n"))
parsed = parseTextReplyBlocks(txt)
self.assertEquals(1, len(parsed))
self.assertEquals(reply.pack(), parsed[0].pack())
@@ -1894,10 +1895,10 @@
def fails(s, p=parseTextReplyBlocks, self=self):
self.assertRaises(ParseError, p, s)
- fails("== BEGIN TYPE III REPLY BLOCK ==\n"+
- "Version: 0.1\n"+
+ fails("-----BEGIN TYPE III REPLY BLOCK-----\n"+
+ "Version: 0.2\n\n"+
"xyz\n"+
- "== END TYPE III REPLY BLOCK ==\n")
+ "-----END TYPE III REPLY BLOCK-----\n")
# Test decoding
seed = loc[:20]
@@ -4345,14 +4346,14 @@
####
# Tests escapeMessageForEmail
self.assert_(stringContains(eme(FDPFast('plain',message)), message))
- expect = "BEGINS =======\nMessage-type: binary\n\n"+\
- encodeBase64(binmessage)+"====="
+ expect = "-----\nMessage-type: binary\n\n"+\
+ encodeBase64(binmessage)+"-----"
self.assert_(stringContains(eme(FDPFast('plain',binmessage)), expect))
- expect = "BEGINS =======\nDecoding-handle: "+\
+ expect = "-----\nMessage-type: encrypted\nDecoding-handle: "+\
base64.encodestring(tag)+"\n"+\
- encodeBase64(binmessage)+"====="
+ encodeBase64(binmessage)+"-----"
self.assert_(stringContains(eme(FDPFast('enc',binmessage,tag)),
- expect))
+ expect))
# Sample address file for testing MBOX
MBOX_ADDRESS_SAMPLE = """\
@@ -4378,14 +4379,15 @@
This message is not in plaintext. It's either 1) a reply; 2) a forward
message encrypted to you; or 3) junk.
-======= TYPE III ANONYMOUS MESSAGE BEGINS =======
+-----BEGIN TYPE III ANONYMOUS MESSAGE-----
+Message-type: encrypted
Decoding-handle: eHh4eHh4eHh4eHh4eHh4eHh4eHg=
7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3u/6zqse+sre7/rOqx76yt7v+s6rHvrK3u/6zqse+sre
7/rOqx76yt7v+s6rHvrK3g==
-======== TYPE III ANONYMOUS MESSAGE ENDS ========
+-----END TYPE III ANONYMOUS MESSAGE-----
"""
EXAMPLE_ADDRESS_SET = """
@@ -4591,11 +4593,13 @@
Avast ye mateys! Prepare to be anonymized!
-======= TYPE III ANONYMOUS MESSAGE BEGINS =======
+-----BEGIN TYPE III ANONYMOUS MESSAGE-----
+Message-type: plaintext
+
Hidden, we are free
Free to speak, to free ourselves
Free to hide no more.
-======== TYPE III ANONYMOUS MESSAGE ENDS ========\n"""
+-----END TYPE III ANONYMOUS MESSAGE-----\n"""
d = findFirstDiff(EXPECTED_SMTP_PACKET, args[3])
if d != -1:
print d, "near", repr(args[3][d-10:d+10])