[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Use Zooko"s zlibutil to do bounded uncompression with P...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv4240/lib/mixminion
Modified Files:
.cvsignore BuildMessage.py Common.py benchmark.py
Log Message:
Use Zooko's zlibutil to do bounded uncompression with Python < 2.2.
Add a hack to setup.py to support multiple builds at once.
Index: .cvsignore
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/.cvsignore,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- .cvsignore 16 Dec 2002 19:17:46 -0000 1.3
+++ .cvsignore 31 Dec 2002 17:40:54 -0000 1.4
@@ -2,3 +2,4 @@
*.pyo
_unittest.py
_textwrap.py
+_zlibutil.py
Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- BuildMessage.py 29 Dec 2002 21:00:18 -0000 1.25
+++ BuildMessage.py 31 Dec 2002 17:40:54 -0000 1.26
@@ -6,12 +6,16 @@
Code to construct messages and reply blocks, and to decode received
message payloads."""
+import sys
import zlib
import operator
import mixminion.Crypto as Crypto
from mixminion.Packet import *
from mixminion.Common import MixError, MixFatalError, LOG
+if sys.version[:3] < (2,2,0):
+ import mixminion._zlibutil as zlibutil
+
__all__ = ['buildForwardMessage', 'buildEncryptedMessage', 'buildReplyMessage',
'buildReplyBlock', 'decodePayload' ]
@@ -648,20 +652,34 @@
if len(payload) < 6 or payload[0:2] != '\x78\xDA':
raise ParseError("Invalid zlib header")
+
+ # This code is necessary because versions of Python before 2.2 didn't
+ # support limited-size versions of zlib.decompress. We use a helper
+ # function helpfully submitted by Zooko.
+ if sys.version_info[:3] < (2,2,0) and maxLength is not None:
+ try:
+ return zlibutil.safe_zlib_decompress_to_retval(payload,
+ maxLength,
+ max(maxLength*3, 1<<20))
+ except zlibutil.TooBigError:
+ raise CompressedDataTooLong()
+ except zlibutil.DecompressError, e:
+ raise ParseError("Error in compressed data")
+
try:
- # We can't just call zlib.decompress(payload), since we'll eventually
+ # We can't just call zlib.decompress(payload), since we may
# want to limit the output size.
+
zobj = zlib.decompressobj(zlib.MAX_WBITS)
# Decompress the payload.
if maxLength is None:
d = zobj.decompress(payload)
else:
- #XXXX Arg! The 'maxlength' argument to decompress wasn't
- #XXXX introduced until Python 2.2. I'll need to fall back
- #XXXX on blunter means.
+ # If we _do_ have Python 2.2, this is the easy way to do it. It
+ # also uses less RAM in the failing case.
d = zobj.decompress(payload, maxLength)
- if zobj.unconsumed_tail:
- raise CompressedDataTooLong()
+ if zobj.unconsumed_tail:
+ raise CompressedDataTooLong()
# Get any leftovers, which shouldn't exist.
nil = zobj.flush()
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.38
retrieving revision 1.39
diff -u -d -r1.38 -r1.39
--- Common.py 31 Dec 2002 04:48:46 -0000 1.38
+++ Common.py 31 Dec 2002 17:40:54 -0000 1.39
@@ -652,7 +652,7 @@
this set."""
if isinstance(other, IntervalSet):
return self*other == other
- idx = bisect.bisect_right(self.edges, (other, '-'))
+ idx = bisect.bisect(self.edges, (other, '-'))
return idx < len(self.edges) and self.edges[idx][1] == '-'
def isEmpty(self):
Index: benchmark.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/benchmark.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- benchmark.py 31 Dec 2002 04:48:47 -0000 1.21
+++ benchmark.py 31 Dec 2002 17:40:54 -0000 1.22
@@ -19,7 +19,8 @@
from time import time
import mixminion._minionlib as _ml
-from mixminion.BuildMessage import _buildHeader, buildForwardMessage
+from mixminion.BuildMessage import _buildHeader, buildForwardMessage, \
+ compressData, uncompressData
from mixminion.Common import secureDelete, installSignalHandlers, \
waitForChildren, formatBase64
from mixminion.Crypto import *
@@ -403,6 +404,26 @@
def buildMessageTiming():
print "#================= BUILD MESSAGE ====================="
pk = pk_generate()
+
+ for payload in "Hello!!!"*128, "Hello!!!"*(128*28):
+ print "Compress %sK" % (len(payload)/1024), \
+ timeit(lambda p=payload: compressData(p),
+ 100)
+
+ compressed = compressData("Hello!!!"*128)
+ print "Uncompress (1K, no max)", \
+ timeit(lambda c=compressed: uncompressData(c), 1000)
+ compressed = compressData("Hello!!!"*(128*28))
+ print "Unompress (28K, no max)", \
+ timeit(lambda c=compressed: uncompressData(c), 1000)
+
+ compressed = compressData("Hello!!!"*128)
+ print "Uncompress (1K, 1K max)", \
+ timeit(lambda c=compressed: uncompressData(c, 1024), 1000)
+ compressed = compressData("Hello!!!"*(128*28))
+ print "Unompress (28K, 28K max)", \
+ timeit(lambda c=compressed: uncompressData(c, 28<<10), 1000)
+
payload = ("Junky qoph flags vext crwd zimb."*1024)[:22*1024]
serverinfo = [FakeServerInfo("127.0.0.1", 48099, pk,"x"*20)
] * 16
@@ -437,6 +458,7 @@
bm(8,1,40)
bm(8,8,20)
bm(16,16,10)
+
#----------------------------------------------------------------------
class DummyLog:
def seenHash(self,h): return 0
@@ -656,4 +678,3 @@
timeEfficiency()
#import profile
#profile.run("import mixminion.benchmark; mixminion.benchmark.directoryTiming()")
-