[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] All known tasks for 0.0.5 are done. Only testing is le...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv16597/lib/mixminion
Modified Files:
BuildMessage.py ClientMain.py Common.py Fragments.py Packet.py
test.py
Log Message:
All known tasks for 0.0.5 are done. Only testing is left before 0.0.5rc1.
BuildMessage, ClientMain:
- Debug SURBLog
- Return SURBLog backward compatibility.
Packet:
- Debug ServerSideFragmentedMessage.pack
test:
- Add tests for random-length paths.
- Tests for clearable queues.
- Tests for reassembly module.
- Tests for maximum message sizes.
- Tests for paths with filenames containing commas and colons.
- Tests for SURB logs
Module:
- Activate modules in a fixed order.
ServerKeys:
- Don't crash when the directory server is down.
ServerMain:
- Even in TRACE mode, don't link incoming messages to pool messages.
*:
- Add all waiting documentation, resolve all XXXX005's.
Index: BuildMessage.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/BuildMessage.py,v
retrieving revision 1.57
retrieving revision 1.58
diff -u -d -r1.57 -r1.58
--- BuildMessage.py 28 Aug 2003 01:40:07 -0000 1.57
+++ BuildMessage.py 31 Aug 2003 19:29:29 -0000 1.58
@@ -291,7 +291,7 @@
# Maybe we shouldn't even allow this to be called with userKey==None.
def buildReplyBlock(path, exitType, exitInfo, userKey,
- expiryTime=0, secretRNG=None):
+ expiryTime=None, secretRNG=None):
"""Construct a 'state-carrying' reply block that does not require the
reply-message recipient to remember a list of secrets.
Instead, all secrets are generated from an AES counter-mode
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.111
retrieving revision 1.112
diff -u -d -r1.111 -r1.112
--- ClientMain.py 28 Aug 2003 18:43:44 -0000 1.111
+++ ClientMain.py 31 Aug 2003 19:29:29 -0000 1.112
@@ -130,6 +130,7 @@
# Mixminion 0.0.1 used an obsolete directory-full-of-servers in
# DIR/servers. If there's nothing there, we remove it. Otherwise,
# we warn.
+ # XXXX010 Eventually, we can remove this.
sdir = os.path.join(self.dir,"servers")
if os.path.exists(sdir):
if os.listdir(sdir):
@@ -761,17 +762,19 @@
As an abbreviation, you can use star followed by a number to indicate
that number of randomly chosen servers:
'foo,bar,*2,quux'.
-
- You can use a star to specify a fill point where randomly-selected
- servers will be added:
+ You can use a star without a number to specify a fill point
+ where randomly-selected servers will be added:
'foo,bar,*,quux'.
+ Finally, you can use a tilde followed by a number to specify an
+ approximate number of servers to add. (The actual number will be
+ chosen randomly, according to a normal distribution with standard
+ deviation 1.5):
+ 'foo,bar,~2,quux'
The nHops argument must be consistent with the path, if both are
specified. Specifically, if nHops is used _without_ a star on the
path, nHops must equal the path length; and if nHops is used _with_ a
star on the path, nHops must be >= the path length.
-
- DOCDOC ~
"""
if not path:
path = '*'
@@ -780,7 +783,6 @@
# or "<swap>"
# or "?"
p = []
- #XXXX005 test 'filename:with:colons',b,c
while path:
if path[0] == "'":
m = re.match(r"'([^']+|\\')*'", path)
@@ -1257,7 +1259,7 @@
def isSURBUsed(self, surb):
"""Return true iff the ReplyBlock object 'surb' is marked as used."""
- return self.has_key[surb]
+ return self.has_key(surb)
def markSURBUsed(self, surb):
"""Mark the ReplyBlock object 'surb' as used."""
@@ -1281,7 +1283,7 @@
self.sync()
def _encodeKey(self, surb):
- return sha1(surb.pack())
+ return binascii.b2a_hex(sha1(surb.pack()))
def _encodeVal(self, timestamp):
return str(timestamp)
def _decodeVal(self, timestamp):
@@ -1297,10 +1299,8 @@
tell us not to."""
## Fields:
# dir -- a directory to store packets in.
- # prng -- an instance of mixminion.Crypto.RNG.
- ## Format:
- # The directory holds files with names of the form pkt_<handle>.
- # Each file holds pickled tuple containing:
+ # store -- an instance of ObjectStore. The entries are of the
+ # format:
# ("PACKET-0",
# a 32K string (the packet),
# an instance of IPV4Info (the first hop),
@@ -1308,18 +1308,15 @@
# packet was inserted into the queue
# )
# XXXX change this to be OO; add nicknames.
-
- # XXXX write unit tests
- #
- # DOCDOC fields have changed.
-
+ # XXXX006 write unit tests
def __init__(self, directory, prng=None):
"""Create a new ClientQueue object, storing packets in 'directory'
and generating random filenames using 'prng'."""
self.dir = directory
createPrivateDir(directory)
+
# We used to name entries "pkt_X"; this has changed.
- #
+ # XXXX006 remove this when it's no longer needed.
for fn in os.listdir(directory):
if fn.startswith("pkt_"):
handle = fn[4:]
@@ -1510,10 +1507,8 @@
#XXXX006 we need to factor this long-message logic out to the
#XXXX006 common code. For now, this is a temporary measure.
-
- # DOCDOC
- fragmentedMessagePrefix = struct.pack("!HH", routingType,
- len(routingInfo))+routingInfo
+ fragmentedMessagePrefix = mixminion.Packet.ServerSideFragmentedMessage(
+ routingType, routingInfo, "").pack()
LOG.info("Generating payload(s)...")
r = []
payloads = mixminion.BuildMessage.encodeMessage(message, 0,
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.108
retrieving revision 1.109
diff -u -d -r1.108 -r1.109
--- Common.py 28 Aug 2003 08:39:21 -0000 1.108
+++ Common.py 31 Aug 2003 19:29:29 -0000 1.109
@@ -1522,10 +1522,8 @@
class ClearableQueue(MessageQueue):
"""Extended version of python's Queue class that supports removing
all the items from the queue."""
- #XXXX005 testme
def clear(self):
"""Remove all the items from this queue."""
-
# If the queue is empty, return.
if not self.esema.acquire(0):
return
Index: Fragments.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Fragments.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- Fragments.py 28 Aug 2003 01:40:07 -0000 1.8
+++ Fragments.py 31 Aug 2003 19:29:29 -0000 1.9
@@ -131,7 +131,7 @@
self.rescan()
def cleanQueue(self, deleteFn=None):
- """DOCDOC"""
+ """Expunge all removed fragments from disk. See Filestore.cleanQueue"""
self.store.cleanQueue(deleteFn)
def sync(self):
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.59
retrieving revision 1.60
diff -u -d -r1.59 -r1.60
--- Packet.py 28 Aug 2003 18:43:44 -0000 1.59
+++ Packet.py 31 Aug 2003 19:29:29 -0000 1.60
@@ -421,8 +421,10 @@
self.routinginfo = routinginfo
self.compressedContents = compressedContents
def pack(self):
- return struct.pack(SSF_UNPACK_PATTERN, self.routingtype,
- len(self.routinginfo)) + self.compressedContents
+ return "%s%s%s" % (struct.pack(SSF_UNPACK_PATTERN, self.routingtype,
+ len(self.routinginfo)),
+ self.routinginfo,
+ self.compressedContents)
#----------------------------------------------------------------------
# REPLY BLOCKS
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.149
retrieving revision 1.150
diff -u -d -r1.149 -r1.150
--- test.py 28 Aug 2003 01:40:08 -0000 1.149
+++ test.py 31 Aug 2003 19:29:29 -0000 1.150
@@ -180,7 +180,7 @@
if s1 != s2:
d = findFirstDiff(s1, s2)
self.fail("Strings unequal. First difference at %s: %r vs %r"
- % (d, s1[d:+10], s2[d:d+10]))
+ % (d, s1[d:d+10], s2[d:d+10]))
def assertUnorderedEq(self, l1, l2):
"""Fail unless the lists l1 and l2 have the same elements. The
order of elements in the two lists need not be the same."""
@@ -671,6 +671,19 @@
"-----BEGIN X-----\n"
"A: X\n\n"
"A B C\n-----END X-----\n", ["X"], 1)
+
+ def test_clearableQueue(self):
+ import Queue
+ #FFFF This test is inadequate for weird multithreaded
+ q = mixminion.Common.ClearableQueue()
+ self.assert_(q.empty())
+ q.put(1)
+ q.put(2)
+ self.assertEquals(2, q.qsize())
+ self.assertEquals(1, q.get())
+ q.clear()
+ self.assert_(q.empty())
+ self.assertRaises(Queue.Empty, q.get_nowait)
#----------------------------------------------------------------------
@@ -1142,7 +1155,7 @@
v = PRNG.getNormal(5,1)
self.failUnless(0 <= v <= 10)
tot += v
- self.failUnless(4500<tot<5500)
+ self.failUnless(4900<tot<5100)
## itot=ftot=0
## for i in xrange(1000000):
@@ -4179,6 +4192,10 @@
Enabled: no
Retry: every 1 hour for 1 day, every 1 day for 1 week
+[Delivery/Fragmented]
+Enabled yes
+MaximumSize: 100k
+
"""
SERVER_CONFIG_SHORT = """
@@ -4238,6 +4255,10 @@
0,65535),
])
eq(info['Delivery/MBOX'].get('Version'), None)
+
+ eq(info['Delivery/Fragmented'].get('Version'), '0.1')
+ eq(info['Delivery/Fragmented'].get('Maximum-Fragments'), 6)
+
# Check the more complex helpers.
self.assert_(info.isValidated())
self.assertEquals(info.getIntervalSet(),
@@ -4279,6 +4300,8 @@
self.assert_(info.isNewerThan(time.time()-60*60))
self.assert_(not info.isNewerThan(time.time()+60))
+ self.assertUnorderedEq(info.getCaps(), ["relay", "frag"])
+
# Now check whether we still validate the same after some corruption
self.assertStartsWith(inf, "[Server]\n")
self.assertEndsWith(inf, "\n")
@@ -5120,7 +5143,6 @@
This is the message
-----END TYPE III ANONYMOUS MESSAGE-----
''')
-
finally:
undoReplacedAttributes()
clearReplacedFunctionCallLog()
@@ -5140,6 +5162,7 @@
Message: Avast ye mateys! Prepare to be anonymized!
ReturnAddress: yo.ho.ho@bottle.of.rum
SubjectLine: Arr! This be a Type III Anonymous Message
+MaximumSize: 35K
""" % blacklistFile)
module = manager.nameToModule["SMTP"]
@@ -5242,6 +5265,19 @@
self.assertEquals(1,s.count(
"Dropping SMTP message to invalid address 'not.an.addr'"))
self.assertEquals([], getReplacedFunctionCallLog())
+
+
+ # Message over 35K should be blocked.
+ m = Crypto.getCommonPRNG().getBytes(6*1024)*6
+ self.assert_(len(BuildMessage.compressData(m))<28*1024)
+ try:
+ suspendLog()
+ queueMessage(FDP('plain',SMTP_TYPE,"foo@bar.bax",m))
+ queue.sendReadyMessages()
+ finally:
+ s = resumeLog()
+ self.assertEquals(1, s.count("Dropping over-long message"))
+ self.assertEquals([], getReplacedFunctionCallLog())
finally:
undoReplacedAttributes()
clearReplacedFunctionCallLog()
@@ -5389,6 +5425,73 @@
self.assertEquals(1, queue.count())
self.assertEquals(5, len(os.listdir(dir)))
+ def testFragmented(self):
+
+ manager = self.getManager("""[Delivery/SMTP]
+Enabled: yes
+SMTPServer: nowhere
+Message: Avast ye mateys! Prepare to be anonymized!
+ReturnAddress: yo.ho.ho@bottle.of.rum
+SubjectLine: Arr! This be a Type III Anonymous Message
+MaximumSize: 1M
+[Delivery/Fragmented]
+Enabled: yes
+MaximumSize: 1M
+""")
+ message = Crypto.getCommonPRNG().getBytes(200*1024)
+ hm = mixminion.Packet.encodeMailHeaders(subject="Hello")
+ payloads = BuildMessage.encodeMessage(hm+message, 0,
+ mixminion.Packet.ServerSideFragmentedMessage(SMTP_TYPE,
+ "pirates@sea","").pack())
+ deliv = [ mixminion.server.PacketHandler.DeliveryPacket(
+ routingType=FRAGMENT_TYPE, routingInfo="", applicationKey="X"*16,
+ tag="", payload=p) for p in payloads ]
+ self.assertEquals(len(deliv), 11)
+
+ replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
+ lambda *args: mixminion.server.Modules.DELIVER_OK)
+ try:
+ # 6 packets; not enough to reconstruct.
+ for p in deliv[:6]:
+ manager.queueDecodedMessage(p)
+ manager.sendReadyMessages()
+ self.assertEquals(getReplacedFunctionCallLog(), [])
+ manager.queueDecodedMessage(deliv[9])
+ manager.queueDecodedMessage(deliv[10])
+ manager.sendReadyMessages()
+ # 8 packets; should be enough to reconstruct.
+ calls = getReplacedFunctionCallLog()
+ self.assertEquals(1, len(calls))
+ fn, args, _ = calls[0]
+ self.assertEquals(fn, "sendSMTPMessage")
+ self.assertEquals(("nowhere", ["pirates@sea"],
+ "yo.ho.ho@bottle.of.rum"), args[:3])
+ self.assertLongStringEq(args[3], """\
+To: pirates@sea
+From: yo.ho.ho@bottle.of.rum
+Subject: Hello
+X-Anonymous: yes
+
+Avast ye mateys! Prepare to be anonymized!
+
+This message contains nonprinting characters, so I encoded it with Base64
+before sending it to you.
+
+-----BEGIN TYPE III ANONYMOUS MESSAGE-----
+Message-type: binary
+
+%s-----END TYPE III ANONYMOUS MESSAGE-----
+""" % encodeBase64(message))
+
+ pool = manager.queues["FRAGMENT"].pool
+ self.assertEquals(pool.store.count(), 0)
+
+ manager.queueDecodedMessage(deliv[3])
+ self.assertEquals(pool.store.count(), 0)
+ finally:
+ undoReplacedAttributes()
+ clearReplacedFunctionCallLog()
+
def getManager(self, extraConfig=None):
d = mix_mktemp()
c = SERVER_CONFIG_SHORT % d
@@ -6032,6 +6135,18 @@
eq((len(p1),len(p2)), (2,3))
pathIs((p1[:1],p2[-2:]), ((alice,),(bob,lola)))
+ # 1a'. Filename with internal commas and colons, where permitted.
+ fredfile2 = os.path.join(mix_mktemp(), "a:b,c")
+ try:
+ writeFile(fredfile2,edesc["Fred"][1])
+ except OSError:
+ pass
+ else:
+ p1,p2 = ppath(ks, None, "Alice,%r,Bob,Joe"%fredfile2, email)
+ pathIs(p1,p2), ((alice,fred),(bob,joe))
+ p1,p2 = ppath(ks, None, "%r,Alice,Bob,Joe"%fredfile2, email)
+ pathIs(p1,p2), ((fred,alice),(bob,joe))
+
# 1b. Colon, no star
p1,p2 = ppath(ks, None, "Alice:Fred,Joe", email)
pathIs((p1,p2), ((alice,),(fred,joe)))
@@ -6107,6 +6222,11 @@
p1,p2 = ppath(ks, None, '?,~4,Bob,Joe', email) #default nHops=6
p = p1+p2
pathIs((p2[-1], p2[-2],), (joe, bob))
+ total = 0
+ for _ in xrange(1000):
+ p1,p2 = ppath(ks, None, '~2,Bob,Joe', email) #default nHops=6
+ total += len(p1+p2)
+ self.assert_(3.4 <= total/1000.0 <= 4.6)
# 1e. Complex.
try:
@@ -6197,13 +6317,38 @@
parseFails("0xFEEEF:zymurgy") # Hex literal out of range
def testSURBLog(self):
+ brb = BuildMessage.buildReplyBlock
SURBLog = mixminion.ClientMain.SURBLog
+ ServerInfo = mixminion.ServerInfo.ServerInfo
dirname = mix_mktemp()
fname = os.path.join(dirname, "surblog")
+
+ # generate 3 SURBs.
+ examples = getExampleServerDescriptors()
+ alice = ServerInfo(string=examples["Alice"][0])
+ lola = ServerInfo(string=examples["Lola"][0])
+ joe = ServerInfo(string=examples["Joe"][0])
+ surbs = [brb([alice,lola,joe], SMTP_TYPE, "bjork@iceland", "x",
+ time.time()+24*60*60)
+ for _ in range(3)]
+
+ #FFFF check for skipping expired and shortlived SURBs.
+
s = SURBLog(fname)
try:
- #XXXX005 writeme
- pass
+ self.assert_(not s.isSURBUsed(surbs[0]))
+ self.assert_(not s.isSURBUsed(surbs[1]))
+ s.markSURBUsed(surbs[0])
+ self.assert_(s.isSURBUsed(surbs[0]))
+ s.close()
+ s = SURBLog(fname)
+ self.assert_(s.isSURBUsed(surbs[0]))
+ self.assert_(not s.isSURBUsed(surbs[1]))
+ self.assert_(s.findUnusedSURB(surbs) is surbs[1])
+ s.markSURBUsed(surbs[1])
+ self.assert_(s.findUnusedSURB(surbs) is surbs[2])
+ s.markSURBUsed(surbs[2])
+ self.assert_(s.findUnusedSURB(surbs) is None)
finally:
s.close()
@@ -6569,7 +6714,7 @@
tc = loader.loadTestsFromTestCase
if 0:
- suite.addTest(tc(BuildMessageTests))
+ suite.addTest(tc(ModuleTests))
return suite
testClasses = [MiscTests,
MinionlibCryptoTests,