[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Debug client-side fragment reassembly.
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv5478/lib/mixminion
Modified Files:
ClientMain.py ClientUtils.py Common.py Fragments.py Main.py
Log Message:
Debug client-side fragment reassembly.
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.161
retrieving revision 1.162
diff -u -d -r1.161 -r1.162
--- ClientMain.py 2 Mar 2004 06:05:32 -0000 1.161
+++ ClientMain.py 2 Mar 2004 07:06:14 -0000 1.162
@@ -1989,7 +1989,7 @@
purge = 1
if not args:
- print "No message-IDs provided."
+ print "No message IDs provided."
return
parser.init()
@@ -1999,16 +1999,18 @@
if reassemble:
out = sys.stdout
if outfilename not in ('-',None):
- out = open(outfilename, 'r')
+ out = open(outfilename, 'wb')
closeoutfile = 1
try:
clientLock()
+ removed = []
for msgid in args:
if reassemble:
msg = client.pool.getMessage(msgid)
if purge:
- client.pool.removeMessage(msgid)
+ removed.append(msgid)
+ client.pool.removeMessages(removed)
if reassemble:
out.write(msg)
finally:
Index: ClientUtils.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientUtils.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- ClientUtils.py 2 Mar 2004 06:05:32 -0000 1.22
+++ ClientUtils.py 2 Mar 2004 07:06:14 -0000 1.23
@@ -25,7 +25,7 @@
from mixminion.Common import LOG, MixError, UIError, ceilDiv, \
createPrivateDir, floorDiv, previousMidnight, readFile, \
- succeedingMidnight, writeFile, armorText, unarmorText
+ succeedingMidnight, writeFile, armorText, unarmorText, MixFatalError
from mixminion.Crypto import sha1, ctr_crypt, DIGEST_LEN, AES_KEY_LEN, \
getCommonPRNG, trng
@@ -869,7 +869,7 @@
def __getPool(self):
if self.pool is None:
import mixminion.Fragments
- self.pool = mixminion.Fragments.FragmentPool(self.directory)
+ self.pool = mixminion.Fragments.FragmentPool(self.dir)
return self.pool
def close(self):
@@ -886,12 +886,14 @@
fragment = mixminion.Packet.parsePayload(fragment)
except ParseError, s:
raise UIError("Corrupted fragment payload: %s"%s)
- if not fragment.isFragment():
+ if fragment.isSingleton():
raise UIError("Non-fragment payload marked as a fragment.")
assert isinstance(fragment, mixminion.Packet.FragmentPayload)
- return pool.addFragment(fragment, nym=nym, verbose=1)
+ r = pool.addFragment(fragment, nym=nym, verbose=1)
+ pool.unchunkMessages(); print "UNCHUNK"
+ return r
def process(self):
pool = self.__getPool()
@@ -905,24 +907,28 @@
def getMessage(self, msgid):
pool = self.__getPool()
- msg = pool.getReadyMessage(msgid)
+ state = pool.getStateByMsgID(msgid)
+ msg = pool.getReadyMessage(state.messageid)
if msg is not None:
return msg
- state = pool.getStateByMsgID(msgid)
if state is None:
raise UIError("No such message as '%s'" % msgid)
elif not state.isDone():
raise UIError("Message '%s' is still missing fragments."%msgid)
else:
- raise MixFatalError("Can't decode message %s; I don't know why!")
+ raise MixFatalError("Can't decode message %s; I don't know why!"
+ %msgid)
def removeMessages(self, msgids):
pool = self.__getPool()
+ idSet = {}
for i in msgids:
- if pool.getStateByMsgID(m) is None:
+ state = pool.getStateByMsgID(i)
+ if state is None:
raise UIError("No such message as %s")
- pool._deleteMessageIDs(msgids, "?")
+ idSet[state.messageid] = 1
+ pool._deleteMessageIDs(idSet, "?")
pool.cleanQueue()
def listMessages(self):
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.131
retrieving revision 1.132
diff -u -d -r1.131 -r1.132
--- Common.py 21 Feb 2004 00:02:09 -0000 1.131
+++ Common.py 2 Mar 2004 07:06:14 -0000 1.132
@@ -318,7 +318,7 @@
if base64:
try:
if stringContains(s[idx:endIdx], "\n[...]\n"):
- raise ValueError("Value seems to be truncated by a Mixminion-Mixmaster gateway")
+ raise UIError("Corrupted data: value seems to be truncated by a Mixminion/Mixmaster gateway")
value = binascii.a2b_base64(s[idx:endIdx])
except (TypeError, binascii.Incomplete, binascii.Error), e:
raise ValueError(str(e))
Index: Fragments.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Fragments.py,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- Fragments.py 16 Feb 2004 22:30:03 -0000 1.12
+++ Fragments.py 2 Mar 2004 07:06:14 -0000 1.13
@@ -10,7 +10,7 @@
import time
import mixminion._minionlib
import mixminion.Filestore
-from mixminion.Crypto import ceilDiv, getCommonPRNG, whiten, unwhiten
+from mixminion.Crypto import ceilDiv, getCommonPRNG, sha1, whiten, unwhiten
from mixminion.Common import disp64, LOG, previousMidnight, MixError, \
MixFatalError
from mixminion.Packet import ENC_FWD_OVERHEAD, PAYLOAD_LEN, \
@@ -183,7 +183,8 @@
chunkNum=None,
overhead=fragmentPacket.getOverhead(),
insertedDate=today,
- nym=nym)
+ nym=nym,
+ digest=sha1(fragmentPacket.data))
# ... and allocate or find the MessageState for this message.
state = self._getState(meta)
try:
@@ -195,19 +196,19 @@
# And *now* update the message state.
state.addFragment(h, meta)
say("Stored fragment %s of message %s",
- fragmentPacket.index, disp64(fragmentPacket.msgID,12))
+ fragmentPacket.index+1, disp64(fragmentPacket.msgID,12))
return fragmentPacket.msgID
except MismatchedFragment, s:
# Remove the other fragments, mark msgid as bad.
LOG.warn("Found inconsistent fragment %s in message %s: %s",
- fragmentPacket.index, disp64(fragmentPacket.msgID,12),
+ fragmentPacket.index+1, disp64(fragmentPacket.msgID,12),
s)
self._deleteMessageIDs({ meta.messageid : 1}, "REJECTED", now)
return None
except UnneededFragment:
# Discard this fragment; we don't need it.
- LOG.debug("Dropping unneeded fragment %s of message %s",
- fragmentPacket.index, disp64(fragmentPacket.msgID,12))
+ say("Dropping unneeded fragment %s of message %s",
+ fragmentPacket.index+1, disp64(fragmentPacket.msgID,12))
return None
def getReadyMessage(self, msgid):
@@ -337,8 +338,17 @@
else:
LOG.debug("Removing messages by IDs: %s",
messageIDSet.keys())
+
for mid in messageIDSet.keys():
- self.db.markStatus(mid, why, today)
+ if why == "?":
+ state = self.states[mid]
+ if state.isDone:
+ whythis = "COMPLETED"
+ else:
+ whythis = "REJECTED"
+ else:
+ whythis = why
+ self.db.markStatus(mid, whythis, today)
try:
del self.states[mid]
except KeyError:
@@ -417,8 +427,9 @@
# ENC_FWD_OVERHEAD.
# insertedDate -- Midnight GMT before the day this fragment was received.
# nym -- name of the identity that received this fragment.
+ # digest -- digest of the fragment/chunk; None for pre-0.0.7
def __init__(self, messageid, idx, size, isChunk, chunkNum, overhead,
- insertedDate, nym):
+ insertedDate, nym, digest):
self.messageid = messageid
self.idx = idx
self.size = size
@@ -427,17 +438,23 @@
self.overhead = overhead
self.insertedDate = insertedDate
self.nym = nym
+ self.digest = digest
def __getstate__(self):
- return ("V0", self.messageid, self.idx, self.size,
+ return ("V1", self.messageid, self.idx, self.size,
self.isChunk, self.chunkNum, self.overhead, self.insertedDate,
- self.nym)
+ self.nym, self.digest)
def __setstate__(self, state):
if state[0] == 'V0':
(_, self.messageid, self.idx, self.size,
self.isChunk, self.chunkNum, self.overhead, self.insertedDate,
self.nym) = state
+ self.digest = None
+ elif state[0] == 'V1':
+ (_, self.messageid, self.idx, self.size,
+ self.isChunk, self.chunkNum, self.overhead, self.insertedDate,
+ self.nym,self.digest) = state
else:
raise MixFatalError("Unrecognized fragment state")
@@ -458,7 +475,7 @@
# params -- an instance of FragmentationParams for this message.
# chunks -- a map from chunk number to tuples of (handle within the pool,
# FragmentMetadata object). For completed chunks.
- # fragmentsByChunk -- a map from chunk number to maps from
+ # fragmentsByChunk -- a list mapping chunk number to maps from
# index-within-chunk to (handle,FragmentMetadata)
# readyChunks -- a map whose keys are the numbers of chunks that
# are ready for reconstruction, but haven't been reconstructed
@@ -495,7 +512,7 @@
"""(have,need) DOCDOC"""
need = self.params.k * self.params.nChunks
have = self.params.k * len(self.chunks)
- for d in self.fragmentsByChunk.values():
+ for d in self.fragmentsByChunk:
have += min(len(d),self.params.k)
return have, need
@@ -550,7 +567,11 @@
raise UnneededFragment
if self.fragmentsByChunk[chunkNum].has_key(pos):
- raise MismatchedFragment("multiple fragments for one position")
+ previous = self.fragmentsByChunk[chunkNum][pos]
+ if previous.digest is None or previous.digest == fm.digest:
+ raise UnneededFragment("already seen this fragment")
+ else:
+ raise MismatchedFragment("multiple fragments for one position")
if noop:
return
@@ -585,7 +606,8 @@
idx=chunkno, size=self.params.length,
isChunk=1, chunkNum=chunkno,
overhead=self.overhead,
- insertedDate=minDate, nym=self.nym)
+ insertedDate=minDate, nym=self.nym,
+ digest=sha1(chunkText))
# Queue the chunk.
h2 = store.queueMessageAndMetadata(chunkText, fm2)
del chunkText
Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.70
retrieving revision 1.71
diff -u -d -r1.70 -r1.71
--- Main.py 2 Mar 2004 06:05:32 -0000 1.70
+++ Main.py 2 Mar 2004 07:06:14 -0000 1.71
@@ -131,7 +131,7 @@
"inspect-queue" : ( 'mixminion.ClientMain', 'listQueue' ),
"clean-queue" : ( 'mixminion.ClientMain', 'cleanQueue' ),
"ping" : ( 'mixminion.ClientMain', 'runPing' ),
- "list-fragments" : ( 'mixminion.ClientMain', 'list-fragments' ),
+ "list-fragments" : ( 'mixminion.ClientMain', 'listFragments' ),
"reassemble" : ( 'mixminion.ClientMain', 'reassemble' ),
"purge-fragments" :( 'mixminion.ClientMain', 'reassemble' ),
"server-start" : ( 'mixminion.server.ServerMain', 'runServer' ),