[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Move compression into Packet to avoid annoying circular...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv8847/lib/mixminion
Modified Files:
BuildMessage.py Packet.py benchmark.py test.py
Log Message:
Move compression into Packet to avoid annoying circular depencence.
Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.37
retrieving revision 1.38
diff -u -d -r1.37 -r1.38
--- BuildMessage.py 4 Feb 2003 02:01:46 -0000 1.37
+++ BuildMessage.py 5 Feb 2003 06:28:31 -0000 1.38
@@ -7,7 +7,6 @@
message payloads."""
import sys
-import zlib
import operator
import mixminion.Crypto as Crypto
from mixminion.Packet import *
@@ -619,112 +618,3 @@
'Return true iff the hash on the given payload seems valid'
return payload[2:22] == Crypto.sha1(payload[22:])
-#----------------------------------------------------------------------
-# COMPRESSION FOR PAYLOADS
-
-# Global: contains 0 if we haven't validated zlib; 1 if we have, and 0.5
-# if we're in the middle of validation.
-_ZLIB_LIBRARY_OK = 0
-
-def compressData(payload):
- """Given a string 'payload', compress it with the 'deflate' method
- as specified in the remailer spec and in RFC1951."""
- if not _ZLIB_LIBRARY_OK:
- _validateZlib()
-
- # Don't change any of these options; if different Mixminion clients
- # compress their data differently, an adversary could distinguish
- # messages generated by them.
- zobj = zlib.compressobj(zlib.Z_BEST_COMPRESSION, zlib.DEFLATED,
- zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL,
- zlib.Z_DEFAULT_STRATEGY)
- s1 = zobj.compress(payload)
- s2 = zobj.flush()
- s = s1 + s2
-
- # Now we check the 2 bytes of zlib header. Strictly speaking,
- # these are irrelevant, as are the 4 bytes of adler-32 checksum at
- # the end. Still, we can afford 6 bytes per payload, and
- # reconstructing the checksum to keep zlib happy is a bit of a pain.
- assert s[0] == '\x78' # deflate, 32K window
- assert s[1] == '\xda' # no dict, max compression
- return s
-
-class CompressedDataTooLong(MixError):
- """Exception: raised when try to uncompress data that turns out to be
- longer than we had expected."""
- pass
-
-def uncompressData(payload, maxLength=None):
- """Uncompress a string 'payload'; raise ParseError if it is not
- valid compressed data. If the expanded data is longer than
- maxLength, we raise 'CompressedDataTooLong'."""
-
- if len(payload) < 6 or payload[0:2] != '\x78\xDA':
- raise ParseError("Invalid zlib header")
-
- # This code is necessary because versions of Python before 2.2 didn't
- # support limited-size versions of zlib.decompress. We use a helper
- # function helpfully submitted by Zooko.
- if sys.version_info[:3] < (2,2,0) and maxLength is not None:
- try:
- return zlibutil.safe_zlib_decompress_to_retval(payload,
- maxLength,
- max(maxLength*3, 1<<20))
- except zlibutil.TooBigError:
- raise CompressedDataTooLong()
- except zlibutil.DecompressError, e:
- raise ParseError("Error in compressed data: %s"%e)
-
- try:
- # We can't just call zlib.decompress(payload), since we may
- # want to limit the output size.
-
- zobj = zlib.decompressobj(zlib.MAX_WBITS)
- # Decompress the payload.
- if maxLength is None:
- d = zobj.decompress(payload)
- else:
- # If we _do_ have Python 2.2, this is the easy way to do it. It
- # also uses less RAM in the failing case.
- d = zobj.decompress(payload, maxLength)
- if zobj.unconsumed_tail:
- raise CompressedDataTooLong()
-
- # Get any leftovers, which shouldn't exist.
- nil = zobj.flush()
- if nil != '':
- raise ParseError("Error in compressed data")
- return d
- except zlib.error:
- raise ParseError("Error in compressed data")
-
-def _validateZlib():
- """Internal function: Make sure that zlib is a recognized version, and
- that it compresses things as expected. (This check is important,
- because using a zlib version that compressed differently from zlib1.1.4
- would make senders partitionable by payload compression.)
- """
- global _ZLIB_LIBRARY_OK
- ver = getattr(zlib, "ZLIB_VERSION")
- if ver and ver < "1.1.2":
- raise MixFatalError("Zlib version %s is not supported"%ver)
-
- _ZLIB_LIBRARY_OK = 0.5
- if ver in ("1.1.2", "1.1.3", "1.1.4"):
- _ZLIB_LIBRARY_OK = 1
- return
-
- LOG.warn("Unrecognized zlib version: %r. Spot-checking output", ver)
- # This test is inadequate, but it _might_ catch future incompatible
- # changes.
- _ZLIB_LIBRARY_OK = 0.5
- good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
- '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
- '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
- 'q\x8d\n\x93'
- if compressData("aZbAAcdefg"*1000) == good:
- _ZLIB_LIBRARY_OK = 1
- else:
- _ZLIB_LIBRARY_OK = 0
- raise MixFatalError("Zlib output not as exected.")
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- Packet.py 5 Feb 2003 05:34:55 -0000 1.25
+++ Packet.py 5 Feb 2003 06:28:31 -0000 1.26
@@ -9,7 +9,8 @@
packets, see BuildMessage.py. For functions that handle
server-side processing of packets, see PacketHandler.py."""
-__all__ = [ 'DROP_TYPE', 'ENC_FWD_OVERHEAD', 'ENC_SUBHEADER_LEN',
+__all__ = [ 'compressData', 'CompressedDataTooLong', 'DROP_TYPE',
+ 'ENC_FWD_OVERHEAD', 'ENC_SUBHEADER_LEN',
'FRAGMENT_PAYLOAD_OVERHEAD', 'FWD_TYPE', 'FragmentPayload',
'HEADER_LEN', 'Header', 'IPV4Info', 'MAJOR_NO', 'MBOXInfo',
'MBOX_TYPE', 'MINOR_NO', 'MIN_EXIT_TYPE', 'Message',
@@ -18,20 +19,24 @@
'SMTPInfo', 'SMTP_TYPE', 'SWAP_FWD_TYPE', 'SingletonPayload',
'Subheader', 'TAG_LEN', 'TextEncodedMessage',
'getTotalBlocksForRoutingInfoLen', 'parseHeader', 'parseIPV4Info',
- 'parseMBOXInfo', 'parseMessage', 'parsePayload',
- 'parseReplyBlock', 'parseSMTPInfo', 'parseSubheader',
- 'parseTextEncodedMessage', 'parseTextReplyBlocks'
- ]
+ 'parseMBOXInfo', 'parseMessage', 'parsePayload', 'parseReplyBlock',
+ 'parseSMTPInfo', 'parseSubheader', 'parseTextEncodedMessage',
+ 'parseTextReplyBlocks', 'uncompressData' ]
import base64
import binascii
import re
import struct
+import sys
+import zlib
from socket import inet_ntoa, inet_aton
import mixminion.BuildMessage
from mixminion.Common import MixError, MixFatalError, floorDiv, isSMTPMailbox,\
LOG
+if sys.version_info[:3] < (2,2,0):
+ import mixminion._zlibutil as zlibutil
+
# Major and minor number for the understood packet format.
MAJOR_NO, MINOR_NO = 0,1 #XXXX003 Bump minor_no for 0.0.3
@@ -611,7 +616,7 @@
return TextEncodedMessage(msg, 'BIN'), endIdx
elif msgType == 'LONG':
if force:
- msg = mixminion.BuildMessage.uncompressData(msg) #XXXX may raise
+ msg = uncompressData(msg) #XXXX may raise
return TextEncodedMessage(msg, 'LONG'), endIdx
elif msgType == 'ENC':
tag = binascii.a2b_base64(ascTag)
@@ -664,3 +669,113 @@
return "%s\n%s%s%s%s%s\n" % (
MESSAGE_START_LINE, tagLine, preNL, c, postNL, MESSAGE_END_LINE)
+
+#----------------------------------------------------------------------
+# COMPRESSION FOR PAYLOADS
+
+# Global: contains 0 if we haven't validated zlib; 1 if we have, and 0.5
+# if we're in the middle of validation.
+_ZLIB_LIBRARY_OK = 0
+
+def compressData(payload):
+ """Given a string 'payload', compress it with the 'deflate' method
+ as specified in the remailer spec and in RFC1951."""
+ if not _ZLIB_LIBRARY_OK:
+ _validateZlib()
+
+ # Don't change any of these options; if different Mixminion clients
+ # compress their data differently, an adversary could distinguish
+ # messages generated by them.
+ zobj = zlib.compressobj(zlib.Z_BEST_COMPRESSION, zlib.DEFLATED,
+ zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL,
+ zlib.Z_DEFAULT_STRATEGY)
+ s1 = zobj.compress(payload)
+ s2 = zobj.flush()
+ s = s1 + s2
+
+ # Now we check the 2 bytes of zlib header. Strictly speaking,
+ # these are irrelevant, as are the 4 bytes of adler-32 checksum at
+ # the end. Still, we can afford 6 bytes per payload, and
+ # reconstructing the checksum to keep zlib happy is a bit of a pain.
+ assert s[0] == '\x78' # deflate, 32K window
+ assert s[1] == '\xda' # no dict, max compression
+ return s
+
+class CompressedDataTooLong(MixError):
+ """Exception: raised when try to uncompress data that turns out to be
+ longer than we had expected."""
+ pass
+
+def uncompressData(payload, maxLength=None):
+ """Uncompress a string 'payload'; raise ParseError if it is not
+ valid compressed data. If the expanded data is longer than
+ maxLength, we raise 'CompressedDataTooLong'."""
+
+ if len(payload) < 6 or payload[0:2] != '\x78\xDA':
+ raise ParseError("Invalid zlib header")
+
+ # This code is necessary because versions of Python before 2.2 didn't
+ # support limited-size versions of zlib.decompress. We use a helper
+ # function helpfully submitted by Zooko.
+ if sys.version_info[:3] < (2,2,0) and maxLength is not None:
+ try:
+ return zlibutil.safe_zlib_decompress_to_retval(payload,
+ maxLength,
+ max(maxLength*3, 1<<20))
+ except zlibutil.TooBigError:
+ raise CompressedDataTooLong()
+ except zlibutil.DecompressError, e:
+ raise ParseError("Error in compressed data: %s"%e)
+
+ try:
+ # We can't just call zlib.decompress(payload), since we may
+ # want to limit the output size.
+
+ zobj = zlib.decompressobj(zlib.MAX_WBITS)
+ # Decompress the payload.
+ if maxLength is None:
+ d = zobj.decompress(payload)
+ else:
+ # If we _do_ have Python 2.2, this is the easy way to do it. It
+ # also uses less RAM in the failing case.
+ d = zobj.decompress(payload, maxLength)
+ if zobj.unconsumed_tail:
+ raise CompressedDataTooLong()
+
+ # Get any leftovers, which shouldn't exist.
+ nil = zobj.flush()
+ if nil != '':
+ raise ParseError("Error in compressed data")
+ return d
+ except zlib.error:
+ raise ParseError("Error in compressed data")
+
+def _validateZlib():
+ """Internal function: Make sure that zlib is a recognized version, and
+ that it compresses things as expected. (This check is important,
+ because using a zlib version that compressed differently from zlib1.1.4
+ would make senders partitionable by payload compression.)
+ """
+ global _ZLIB_LIBRARY_OK
+ ver = getattr(zlib, "ZLIB_VERSION")
+ if ver and ver < "1.1.2":
+ raise MixFatalError("Zlib version %s is not supported"%ver)
+
+ _ZLIB_LIBRARY_OK = 0.5
+ if ver in ("1.1.2", "1.1.3", "1.1.4"):
+ _ZLIB_LIBRARY_OK = 1
+ return
+
+ LOG.warn("Unrecognized zlib version: %r. Spot-checking output", ver)
+ # This test is inadequate, but it _might_ catch future incompatible
+ # changes.
+ _ZLIB_LIBRARY_OK = 0.5
+ good = '\x78\xda\xed\xc6A\x11\x00 \x08\x00\xb0l\xd4\xf0\x87\x02\xf6o'+\
+ '`\x0e\xef\xb6\xd7r\xed\x88S=7\xcd\xcc\xcc\xcc\xcc\xcc\xcc'+\
+ '\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xcc\xbe\xdd\x03'+\
+ 'q\x8d\n\x93'
+ if compressData("aZbAAcdefg"*1000) == good:
+ _ZLIB_LIBRARY_OK = 1
+ else:
+ _ZLIB_LIBRARY_OK = 0
+ raise MixFatalError("Zlib output not as exected.")
Index: benchmark.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/benchmark.py,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- benchmark.py 10 Jan 2003 20:12:05 -0000 1.28
+++ benchmark.py 5 Feb 2003 06:28:31 -0000 1.29
@@ -20,14 +20,13 @@
import mixminion._minionlib as _ml
from mixminion.BuildMessage import _buildHeader, buildForwardMessage, \
- compressData, uncompressData, _encodePayload, decodePayload, \
- CompressedDataTooLong
+ compressData, uncompressData, _encodePayload, decodePayload
from mixminion.Common import secureDelete, installSIGCHLDHandler, \
waitForChildren, formatBase64
from mixminion.Crypto import *
from mixminion.Crypto import OAEP_PARAMETER
from mixminion.Crypto import _add_oaep_padding, _check_oaep_padding
-from mixminion.Packet import SMTP_TYPE
+from mixminion.Packet import SMTP_TYPE, CompressedDataTooLong
from mixminion.ServerInfo import ServerInfo
from mixminion.server.HashLog import HashLog
from mixminion.server.PacketHandler import PacketHandler
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.77
retrieving revision 1.78
diff -u -d -r1.77 -r1.78
--- test.py 5 Feb 2003 05:34:55 -0000 1.77
+++ test.py 5 Feb 2003 06:28:31 -0000 1.78
@@ -1289,15 +1289,15 @@
for m in ("", "a", "\000", "xyzzy"*10, ("glossy glossolalia.."*2)[32],
longMsg):
c = BuildMessage.compressData(m)
- self.assertEquals(m, BuildMessage.uncompressData(c))
+ self.assertEquals(m, uncompressData(c))
- self.failUnlessRaises(ParseError, BuildMessage.uncompressData, "3")
+ self.failUnlessRaises(ParseError, uncompressData, "3")
for _ in xrange(20):
for _ in xrange(20):
m = p.getBytes(p.getInt(1000))
try:
- BuildMessage.uncompressData(m)
+ uncompressData(m)
except ParseError:
pass
#FFFF Find a decent test vector.
@@ -1317,7 +1317,7 @@
for ov in 0, 42-20+16: # encrypted forward overhead
pld = BuildMessage._encodePayload(m,ov,p)
self.assertEquals(28*1024, len(pld)+ov)
- comp = BuildMessage.compressData(m)
+ comp = compressData(m)
self.assert_(pld[22:].startswith(comp))
self.assertEquals(sha1(pld[22:]),pld[2:22])
self.assert_(BuildMessage._checkPayload(pld))
@@ -1683,7 +1683,7 @@
self.assertEquals(20, len(t))
self.assertEquals(28*1024,len(p))
self.assertEquals(0, ord(t[0]) & 0x80)
- comp = BuildMessage.compressData("Hello!!!!")
+ comp = compressData("Hello!!!!")
self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
self.assert_(p[22:].startswith(comp))
self.assertEquals(sha1(p[22:]), p[2:22])
@@ -1697,7 +1697,7 @@
ks = Keyset(sessionkey)
msg = rsa_rest + lioness_decrypt(mrest,
ks.getLionessKeys("END-TO-END ENCRYPT"))
- comp = BuildMessage.compressData(payload)
+ comp = compressData(payload)
self.assert_(len(comp), ord(msg[0])*256 + ord(msg[1]))
self.assertEquals(sha1(msg[22:]), msg[2:22])
self.assert_(msg[22:].startswith(comp))
@@ -1818,7 +1818,7 @@
ks = Keyset(s)
p = lioness_encrypt(p, ks.getLionessKeys(
Crypto.PAYLOAD_ENCRYPT_MODE))
- comp = BuildMessage.compressData('Information???')
+ comp = compressData('Information???')
self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
self.assert_(p[22:].startswith(comp))
self.assertEquals(sha1(p[22:]), p[2:22])
@@ -1831,7 +1831,7 @@
ks = Keyset(s)
p = lioness_encrypt(p, ks.getLionessKeys(
Crypto.PAYLOAD_ENCRYPT_MODE))
- comp = BuildMessage.compressData(payload)
+ comp = compressData(payload)
self.assertEquals(len(comp), ord(p[0])*256 +ord(p[1]))
self.assert_(p[22:].startswith(comp))
self.assertEquals(sha1(p[22:]), p[2:22])
@@ -1844,7 +1844,7 @@
"even the invisible victim is responsible for the fate of all.\n"+\
" -- _Invisible Man_"
- comp = BuildMessage.compressData(payload)
+ comp = compressData(payload)
self.assertEquals(len(comp), 109)
encoded1 = (comp+ "RWE/HGW"*4096)[:28*1024-22]
encoded1 = '\x00\x6D'+sha1(encoded1)+encoded1
@@ -1943,7 +1943,7 @@
nils = "\x00"*(25*1024)
overcompressed_payload = \
BuildMessage._encodePayload(nils, 0, AESCounterPRNG())
- self.failUnlessRaises(BuildMessage.CompressedDataTooLong,
+ self.failUnlessRaises(CompressedDataTooLong,
BuildMessage.decodePayload, overcompressed_payload, "X"*20)
# And now the cases that fail hard. This can only happen on:
@@ -2141,7 +2141,7 @@
self.assert_(not pkt.isPlaintext())
self.assert_(pkt.isOvercompressed())
self.assert_(pkt.getAsciiContents(),
- base64.encodestring(BuildMessage.compressData(pcomp)))
+ base64.encodestring(compressData(pcomp)))
m = befm(p, SMTP_TYPE, "nobody@invalid", [self.server1],
[self.server3], getRSAKey(0,1024))
@@ -3886,7 +3886,7 @@
## manager.sendReadyMessages()
## self.assertEquals(len(exampleMod.processedAll), 1)
## self.assertEquals(exampleMod.processedAll[0],
-## (BuildMessage.compressData(p), 'long', 1234, "Buenas noches"))
+## (compressData(p), 'long', 1234, "Buenas noches"))
# Check serverinfo generation.
try: