[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Make fragmentation and reassembly work in practise.
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv28685/lib/mixminion
Modified Files:
BuildMessage.py ClientMain.py Common.py Fragments.py Main.py
Packet.py test.py
Log Message:
Make fragmentation and reassembly work in practise.
BuildMessage, Packet:
- Use symbolic constants instead of '20'.
- Note insanity in exceptions to tag length checking.
BuildMessage:
- Allow decodeMessage to work properly with no tag.
- Make _checkPayload work properly on fragment payloads, instead of rejecting
them all.
ClientMain:
- Fix typo.
Common:
- Add a 'display base 64' function.
Fragments:
- Better logging
- Make backward compatible with Python2.0 syntax.
- Fix bug in getUnneededFragmentHandles
Main:
- Print version on startup.
test:
- More tests for fragmentation
Modules:
- Seriously debug handling of fragmented messages
PacketHandler:
- Handle instances of DeliveryPacket generated by old versions of the
code.
ServerMain:
- Logging tweak.
Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.56
retrieving revision 1.57
diff -u -d -r1.56 -r1.57
--- BuildMessage.py 25 Aug 2003 21:05:33 -0000 1.56
+++ BuildMessage.py 28 Aug 2003 01:40:07 -0000 1.57
@@ -14,6 +14,7 @@
import mixminion.Fragments
from mixminion.Packet import *
from mixminion.Common import MixError, MixFatalError, LOG, UIError
+import mixminion.Packet
import mixminion._minionlib
if sys.version_info[:3] < (2,2,0):
@@ -67,7 +68,7 @@
if uncompressedFragmentPrefix:
payload = uncompressedFragmentPrefix+payload
# Now generate a message ID
- messageid = Crypto.getCommonPRNG().getBytes(20)
+ messageid = Crypto.getCommonPRNG().getBytes(FRAGMENT_MESSAGEID_LEN)
# Figure out how many chunks to divide it into...
p = mixminion.Fragments.FragmentationParams(len(payload), overhead)
# ... fragment the payload into chunks...
@@ -122,7 +123,8 @@
raise MixError("Second leg of path is empty")
suppressTag = 0
- if exitType == DROP_TYPE:
+ #XXXX006 refactor _TYPES_WITHOUT_TAGS
+ if exitType == DROP_TYPE or mixminion.Packet._TYPES_WITHOUT_TAGS.get(exitType):
suppressTag = 1
assert len(payload) == PAYLOAD_LEN
@@ -335,8 +337,10 @@
_getRouting(path1, SWAP_FWD_TYPE, path2[0].getRoutingInfo().pack())
except MixError:
err = 1
- # Add tag as needed to last exitinfo.
- if exitType != DROP_TYPE and exitInfo is not None:
+ # Add a dummy tag as needed to last exitinfo.
+ if (exitType != DROP_TYPE
+ and not mixminion.Packet._TYPES_WITHOUT_TAGS.get(exitType)
+ and exitInfo is not None):
exitInfo += "X"*20
else:
exitInfo = ""
@@ -371,14 +375,20 @@
elif type(userKeys) is types.StringType:
userKeys = { "" : userKeys }
- if len(payload) != PAYLOAD_LEN or len(tag) != TAG_LEN:
- raise MixError("Wrong payload or tag length")
+ if len(payload) != PAYLOAD_LEN:
+ raise MixError("Wrong payload length")
+
+ if len(tag) not in (0, TAG_LEN):
+ raise MixError("Wrong tag length: %s"%len(tag))
# If the payload already contains a valid checksum, it's a forward
# message.
if _checkPayload(payload):
return parsePayload(payload)
+ if not tag:
+ return None
+
# If H(tag|userKey|"Validate") ends with 0, then the message _might_
# be a reply message using H(tag|userKey|"Generate") as the seed for
# its master secrets. (There's a 1-in-256 chance that it isn't.)
@@ -505,7 +515,7 @@
reply = path2
path2 = None
else:
- if len(exitInfo) < TAG_LEN and exitType != DROP_TYPE:
+ if len(exitInfo) < TAG_LEN and exitType != DROP_TYPE and not mixminion.Packet._TYPES_WITHOUT_TAGS.get(exitType):
raise MixError("Implausibly short exit info: %r"%exitInfo)
if exitType < MIN_EXIT_TYPE and exitType != DROP_TYPE:
raise MixError("Invalid exit type: %4x"%exitType)
@@ -700,7 +710,10 @@
def _checkPayload(payload):
'Return true iff the hash on the given payload seems valid'
- return payload[2:22] == Crypto.sha1(payload[22:])
+ if ord(payload[0]) & 0x80:
+ return payload[3:23] == Crypto.sha1(payload[23:])
+ else:
+ return payload[2:22] == Crypto.sha1(payload[22:])
def _getRouting(path, exitType, exitInfo):
"""Given a list of ServerInfo, and a final exitType and exitInfo,
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.108
retrieving revision 1.109
diff -u -d -r1.108 -r1.109
--- ClientMain.py 25 Aug 2003 23:44:30 -0000 1.108
+++ ClientMain.py 28 Aug 2003 01:40:07 -0000 1.109
@@ -1735,11 +1735,12 @@
for msg in parseTextEncodedMessages(s, force=force):
if msg.isOvercompressed() and not force:
LOG.warn("Message is a possible zlib bomb; not uncompressing")
- if msg.isFragment:
+ if msg.isFragment():
raise UIError("Sorry -- no support yet for client-side defragmentation.")
elif not msg.isEncrypted():
results.append(msg.getContents())
else:
+ assert msg.isEncrypted()
surbKeys = self.keys.getSURBKeys()
p = mixminion.BuildMessage.decodePayload(msg.getContents(),
tag=msg.getTag(),
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.106
retrieving revision 1.107
diff -u -d -r1.106 -r1.107
--- Common.py 25 Aug 2003 21:05:33 -0000 1.106
+++ Common.py 28 Aug 2003 01:40:07 -0000 1.107
@@ -9,7 +9,8 @@
'MixError',
'MixFatalError', 'MixProtocolError', 'UIError', 'UsageError',
'armorText', 'ceilDiv', 'checkPrivateDir', 'checkPrivateFile',
- 'createPrivateDir', 'encodeBase64', 'floorDiv', 'formatBase64',
+ 'createPrivateDir', 'disp64',
+ 'encodeBase64', 'floorDiv', 'formatBase64',
'formatDate', 'formatFnameTime', 'formatTime',
'installSIGCHLDHandler', 'isSMTPMailbox', 'openUnique',
'previousMidnight', 'readFile', 'readPickled',
@@ -178,6 +179,15 @@
return "".join([ s.strip() for s in pieces ])
else:
return "".join(pieces)
+
+def disp64(s,n=-1):
+ """Return a 'beautified' base64 for use in log messages."""
+ s = formatBase64(s)
+ if n >= 0:
+ s = s[:n]
+ while s.endswith('='):
+ s = s[:-1]
+ return s
def englishSequence(lst, empty="none"):
"""Given a sequence of items, return the sequence formatted
Index: Fragments.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Fragments.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- Fragments.py 25 Aug 2003 23:44:30 -0000 1.7
+++ Fragments.py 28 Aug 2003 01:40:07 -0000 1.8
@@ -11,7 +11,8 @@
import mixminion._minionlib
import mixminion.Filestore
from mixminion.Crypto import ceilDiv, getCommonPRNG, whiten, unwhiten
-from mixminion.Common import LOG, previousMidnight, MixError, MixFatalError
+from mixminion.Common import disp64, LOG, previousMidnight, MixError, \
+ MixFatalError
from mixminion.Packet import ENC_FWD_OVERHEAD, PAYLOAD_LEN, \
FRAGMENT_PAYLOAD_OVERHEAD
@@ -165,7 +166,7 @@
s = self.db.getStatusAndTime(fragmentPacket.msgID)
if s:
LOG.debug("Dropping fragment of %s message %r",
- s[0].lower(), fragmentPacket.msgID)
+ s[0].lower(), disp64(fragmentPacket.msgID,13))
return
# Otherwise, create a new metadata object for this fragment...
@@ -187,13 +188,17 @@
h = self.store.queueMessageAndMetadata(fragmentPacket.data, meta)
# And *now* update the message state.
state.addFragment(h, meta)
+ LOG.debug("Stored fragment %s of message %s",
+ fragmentPacket.index, disp64(fragmentPacket.msgID,13))
except MismatchedFragment:
# Remove the other fragments, mark msgid as bad.
+ LOG.warn("Found inconsistent fragment %s in message %s",
+ fragmentPacket.index, disp64(fragmentPacket.msgID,13))
self._deleteMessageIDs({ meta.messageid : 1}, "REJECTED", now)
except UnneededFragment:
# Discard this fragment; we don't need it.
- LOG.debug("Dropping unneeded fragment %s of message %r",
- fragmentPacket.index, fragmentPacket.msgID)
+ LOG.debug("Dropping unneeded fragment %s of message %s",
+ fragmentPacket.index, disp64(fragmentPacket.msgID,13))
def getReadyMessage(self, msgid):
"""Return the complete message associated with messageid 'msgid'.
@@ -264,15 +269,16 @@
try:
mid = fm.messageid
if badMessageIDs.has_key(mid):
- # We've already rejected fragments associated with this ID.
- continue
- # All is well; try to register the fragment/chunk. If it's
- # redundant or inconsistent, raise an exception.
- state = self._getState(fm)
- if fm.isChunk:
- state.addChunk(h, fm)
+ # We've already decided to reject fragments with this ID.
+ pass
else:
- state.addFragment(h, fm)
+ # All is well; try to register the fragment/chunk. If it's
+ # redundant or inconsistent, raise an exception.
+ state = self._getState(fm)
+ if fm.isChunk:
+ state.addChunk(h, fm)
+ else:
+ state.addFragment(h, fm)
except MismatchedFragment:
# Mark the message ID for this fragment as inconsistent.
badMessageIDs[mid] = 1
@@ -295,7 +301,7 @@
fm.idx, fm.messageid)
self.store.removeMessage(h)
- # Now nuke inconsistant messages.
+ # Now nuke inconsistent messages.
self._deleteMessageIDs(badMessageIDs, "REJECTED")
def _deleteMessageIDs(self, messageIDSet, why, today=None):
@@ -307,6 +313,8 @@
why -- 'REJECTED' or 'COMPLETED'.
"""
assert why in ("REJECTED", "COMPLETED")
+ if not messageIDSet:
+ return
if today is None:
today = time.time()
today = previousMidnight(today)
@@ -542,7 +550,7 @@
chunks."""
r = []
for chunkno in self.chunks.keys():
- r.extend([ h for h,_ in self.fragmentsByChunk[chunkno]])
+ r.extend([ h for h,_ in self.fragmentsByChunk[chunkno].values()])
return r
class FragmentDB(mixminion.Filestore.DBBase):
Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.56
retrieving revision 1.57
diff -u -d -r1.56 -r1.57
--- Main.py 21 Aug 2003 21:34:02 -0000 1.56
+++ Main.py 28 Aug 2003 01:40:07 -0000 1.57
@@ -203,8 +203,6 @@
sys.exit(1)
def printUsage():
- import mixminion
- print "Mixminion version %s" % mixminion.__version__
print _USAGE
print "NOTE: This software is for testing only. The user set is too small"
print " to be anonymous, and the code is too alpha to be reliable."
@@ -259,6 +257,8 @@
sys.exit(1)
if args[1] not in ('unittests', 'benchmarks', 'version'):
+ import mixminion
+ print "Mixminion version %s" % mixminion.__version__
print "This software is for testing purposes only."\
" Anonymity is not guaranteed."
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.57
retrieving revision 1.58
diff -u -d -r1.57 -r1.58
--- Packet.py 25 Aug 2003 21:05:34 -0000 1.57
+++ Packet.py 28 Aug 2003 01:40:08 -0000 1.58
@@ -13,6 +13,7 @@
'ENC_FWD_OVERHEAD', 'ENC_SUBHEADER_LEN',
'encodeMailHeaders', 'encodeMessageHeaders',
'FRAGMENT_PAYLOAD_OVERHEAD', 'FWD_TYPE', 'FragmentPayload',
+ 'FRAGMENT_MESSAGEID_LEN', 'FRAGMENT_TYPE',
'HEADER_LEN', 'IPV4Info', 'MAJOR_NO', 'MBOXInfo',
'MBOX_TYPE', 'MINOR_NO', 'MIN_EXIT_TYPE',
'MIN_SUBHEADER_LEN', 'Packet',
@@ -83,10 +84,13 @@
SMTP_TYPE = 0x0100 # Mail the message
MBOX_TYPE = 0x0101 # Send the message to one of a fixed list of addresses
NEWS_TYPE = 0x0102 # Post the message to some ngs, and maybe mail it too
-FRAGMENT_TYPE = 0x0103 # Find the actual deliver info in the message payload
+FRAGMENT_TYPE = 0x0103 # Find the actual delivery info in the message payload
MAX_EXIT_TYPE = 0xFFFF
-# Set of exit types that don't get tag fields.
+# Set of exit types that don't get tag fields.
+# XXXX006 This interface is really brittle; it needs to change. I added it
+# XXXX006 in order to allow 'fragment' to be an exit type without adding a
+# XXXX006 needless tag field to every fragment routing info.
_TYPES_WITHOUT_TAGS = { FRAGMENT_TYPE : 1 }
class ParseError(MixError):
@@ -148,7 +152,7 @@
underflow = ""
if rlen < len(ri):
ri, underflow = ri[:rlen], ri[rlen:]
- if rt >= MIN_EXIT_TYPE and not _TYPES_WITHOUT_TAGS.get(rt) and rlen < 20:
+ if rt >= MIN_EXIT_TYPE and not _TYPES_WITHOUT_TAGS.get(rt) and rlen < TAG_LEN:
raise ParseError("Subheader missing tag")
return Subheader(major,minor,secret,digest,rt,ri,rlen,underflow)
@@ -192,6 +196,7 @@
"""Return the part of the routingInfo that contains the delivery
address. (Requires that routingType is an exit type.)"""
assert self.routingtype >= MIN_EXIT_TYPE
+ #XXXX006 This interface is completely insane. Change it.
if _TYPES_WITHOUT_TAGS.get(self.routingtype):
return self.routinginfo
else:
@@ -202,6 +207,7 @@
"""Return the part of the routingInfo that contains the decoding
tag. (Requires that routingType is an exit type.)"""
assert self.routingtype >= MIN_EXIT_TYPE
+ #XXXX006 This interface is completely insane. Change it.
if _TYPES_WITHOUT_TAGS.get(self.routingtype):
return ""
else:
@@ -704,7 +710,7 @@
a tag.
"""
assert messageType in ('TXT', 'ENC', 'LONG', 'BIN', 'FRAG')
- assert tag is None or (messageType == 'ENC' and len(tag) == 20)
+ assert tag is None or (messageType == 'ENC' and len(tag) == TAG_LEN)
self.contents = contents
self.messageType = messageType
self.tag = tag
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.148
retrieving revision 1.149
diff -u -d -r1.148 -r1.149
--- test.py 25 Aug 2003 21:05:34 -0000 1.148
+++ test.py 28 Aug 2003 01:40:08 -0000 1.149
@@ -2221,6 +2221,16 @@
else:
self.assertEquals(decoded, p.getUncompressedContents())
+ def test_fragments(self):
+ msg = Crypto.getCommonPRNG().getBytes(30000)
+ prefix = "Prefix!"
+ payloads = BuildMessage.encodeMessage(msg, 0, prefix)
+ self.assertEquals(len(payloads), 3)
+ p1 = BuildMessage.decodePayload(payloads[0], "")
+ p2 = BuildMessage.decodePayload(payloads[1], "")
+ p3 = BuildMessage.decodePayload(payloads[2], "")
+ self.assert_(None not in [p1, p2, p3])
+
def test_decoding(self):
# Now we create a bunch of fake payloads and try to decode them.
@@ -6525,7 +6535,12 @@
# Change index of message to impossible value, make sure m3 gets
# dropped.
pkts3[60].index = 66
- pool.addFragment(pkts3[60])
+ try:
+ suspendLog()
+ pool.addFragment(pkts3[60])
+ finally:
+ s = resumeLog()
+ self.assert_(stringContains(s, "Found inconsistent fragment"))
self.assertEquals(pool.store.count(), 4+4)
for i in xrange(40,55):
pool.addFragment(pkts3[i])
@@ -6554,7 +6569,7 @@
tc = loader.loadTestsFromTestCase
if 0:
- suite.addTest(tc(FragmentTests))
+ suite.addTest(tc(BuildMessageTests))
return suite
testClasses = [MiscTests,
MinionlibCryptoTests,