[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[minion-cvs] Timeouts, descriptor parsing, refactoring, more tests.
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv22424/lib/mixminion
Modified Files:
ClientMain.py Common.py Config.py MMTPClient.py Packet.py
ServerInfo.py test.py
Log Message:
Timeouts, descriptor parsing, refactoring, more tests.
ClientMain, MMTPClient, Config:
- Support configurable client timeout
Common:
- Make stdout/stderr wrappers not suck
Config, ServerInfo:
- Add support for ServerInfo and directory parsing to ignore unrecognized
keys
Packet, ServerList, ..., ServerKeys:
- Note work for 0.0.3
ServerMain:
- Change ProcessingThread to be callable-based, not packet-based.
test:
- Tests for log streams
- Tests for client timeouts
- Tests for interval list warnings
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -d -r1.43 -r1.44
--- ClientMain.py 13 Jan 2003 06:13:33 -0000 1.43
+++ ClientMain.py 17 Jan 2003 06:18:06 -0000 1.44
@@ -82,6 +82,7 @@
self.__scanning = 0
self.__load()
self.clean()
+ #XXXX003 Check version against directory's Recommended-Software field.
# Mixminion 0.0.1 used an obsolete directory-full-of-servers in
# DIR/servers. If there's nothing there, we remove it. Otherwise,
@@ -789,6 +790,9 @@
[Security]
PathLength: 4
+[Network]
+ConnectionTimeout: 20 seconds
+
""")
f.close()
@@ -844,19 +848,19 @@
"""Given a list of packets and a ServerInfo object, sends the
packets to the server via MMTP"""
LOG.info("Connecting...")
- con = mixminion.MMTPClient.BlockingClientConnection(server.getAddr(),
- server.getPort(),
- server.getKeyID())
+ timeout = self.config['Network'].get('ConnectionTimeout')
+ if timeout:
+ timeout = timeout[2]
+
try:
- try:
- con.connect()
- LOG.info("Sending packet(s)")
- for msg in msgList:
- con.sendPacket(msg)
- except socket.error, e:
- raise MixError("Error sending packets: %s" % e)
- finally:
- con.shutdown()
+ # May raise TimeoutError
+ mixminion.MMTPClient.sendMessages(server.getAddr(),
+ server.getPort(),
+ server.getKeyID(),
+ msgList,
+ timeout)
+ except socket.error, e:
+ raise MixError("Error sending packets: %s" % e)
def parseAddress(s):
"""Parse and validate an address; takes a string, and returns an Address
@@ -1077,7 +1081,7 @@
keystore.updateDirectory(forceDownload=download)
if address is None:
- print "No recipients specified; exiting."
+ print >>sys.stderr, "No recipients specified; exiting."
sys.exit(0)
try:
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- Common.py 10 Jan 2003 20:12:05 -0000 1.52
+++ Common.py 17 Jan 2003 06:18:06 -0000 1.53
@@ -534,8 +534,24 @@
def __init__(self, name, severity):
self.name = name
self.severity = severity
+ self.buf = [] #DOCDOC
def write(self, s):
- LOG.log(self.severity, "->%s: %s", self.name, s)
+ #DOCDOC justify this inefficiency.
+ #LOG.log(self.severity, "called with %r", s)
+ if "\n" not in s:
+ self.buf.append(s)
+ return
+
+ while "\n" in s:
+ idx = s.index("\n")
+ line = "%s%s" %("".join(self.buf), s[:idx])
+ LOG.log(self.severity, "->%s: %s", self.name, line)
+ del self.buf[:]
+ s = s[idx+1:]
+
+ if s:
+ self.buf.append(s)
+
def flush(self): pass
def close(self): pass
@@ -767,7 +783,7 @@
options = os.WNOHANG
while 1:
try:
- # FFFF This won't work on Windows. What to do?
+ # WIN32 This won't work on Windows. What to do?
pid, status = os.waitpid(0, options)
except OSError, e:
break
@@ -784,7 +800,7 @@
while 1:
try:
- # This waitpid call won't work on Windows. What to do?
+ # WIN32 This waitpid call won't work on Windows. What to do?
pid, status = os.waitpid(0, os.WNOHANG)
if pid == 0:
break
Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.35
retrieving revision 1.36
diff -u -d -r1.35 -r1.36
--- Config.py 13 Jan 2003 06:15:10 -0000 1.35
+++ Config.py 17 Jan 2003 06:18:06 -0000 1.36
@@ -497,6 +497,7 @@
_syntax = None
_restrictFormat = 0
+ _restrictKeys = 1 #DOCDOC
def __init__(self, filename=None, string=None, assumeValid=0):
"""Create a new _ConfigFile. If <filename> is set, read from
@@ -582,8 +583,12 @@
try:
rule, parseFn, default = secConfig[k]
except KeyError:
- raise ConfigError("Unrecognized key %s on line %s" %
- (k, line))
+ if self._restrictKeys:
+ raise ConfigError("Unrecognized key %s on line %s" %
+ (k, line))
+ else:
+ LOG.warn("Unregognized key %s on line %s", k, line)
+ continue
# Parse and validate the value of this entry.
if parseFn is not None:
@@ -711,6 +716,7 @@
'SURBAddress' : ('ALLOW', None, None),
'SURBPathLength' : ('ALLOW', _parseInt, "8"),
'SURBLifetime' : ('ALLOW', _parseInterval, "7 days") },
+ 'Network' : { 'ConnectionTimeout' : ('ALLOW', _parseInterval, None) }
}
def __init__(self, fname=None, string=None):
_ConfigFile.__init__(self, fname, string)
@@ -725,6 +731,13 @@
if p < 4:
LOG.warn("Your default path length is frighteningly low."
" I'll trust that you know what you're doing.")
+
+ t = sections['Network'].get('ConnectionTimeout')
+ if t:
+ if t[2] < 5:
+ LOG.warn("Very short connection timeout")
+ elif t[2] > 60:
+ LOG.warn("Very long connection timeout")
def _validateHostSection(sec):
"""Helper function: Makes sure that the shared [Host] section is correct;
Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- MMTPClient.py 14 Jan 2003 09:20:17 -0000 1.17
+++ MMTPClient.py 17 Jan 2003 06:18:06 -0000 1.18
@@ -53,11 +53,11 @@
self.tls = None
self.sock = None
- def connect(self, timeout=None):
+ def connect(self, connectTimeout=None):
"""Negotiate the handshake and protocol."""
def sigalarmHandler(sig, _):
assert sig == signal.SIGALRM
- if timeout:
+ if connectTimeout:
signal.signal(signal.SIGALRM, sigalarmHandler)
# Connect to the server
@@ -66,15 +66,20 @@
LOG.debug("Connecting to %s:%s", self.targetIP, self.targetPort)
# Do the TLS handshaking
- if timeout:
- signal.alarm(timeout)
+ if connectTimeout:
+ signal.alarm(connectTimeout)
try:
- self.sock.connect((self.targetIP,self.targetPort))
- except socket.error, e:
- if e[0] == errno.EINTR:
- raise TimeoutError("Connection timed out")
- else:
- raise e
+ try:
+ self.sock.connect((self.targetIP,self.targetPort))
+ except socket.error, e:
+ if e[0] == errno.EINTR:
+ raise TimeoutError("Connection timed out")
+ else:
+ raise e
+ finally:
+ if connectTimeout:
+ signal.alarm(0)
+
LOG.debug("Handshaking with %s:%s",self.targetIP, self.targetPort)
self.tls = self.context.sock(self.sock.fileno())
# FFFF session resumption
@@ -152,7 +157,8 @@
self.sock.close()
LOG.debug("Connection closed")
-def sendMessages(targetIP, targetPort, targetKeyID, packetList):
+def sendMessages(targetIP, targetPort, targetKeyID, packetList,
+ connectTimeout=None):
"""Sends a list of messages to a server.
DOCDOC arguments
DOCDOC "JUNK", "RENEGOTIATE"
@@ -169,7 +175,7 @@
con = BlockingClientConnection(targetIP, targetPort, targetKeyID)
try:
- con.connect()
+ con.connect(connectTimeout=connectTimeout)
for t,p in packets:
if t == "JUNK":
con.sendJunkPacket(p)
Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- Packet.py 16 Dec 2002 02:40:11 -0000 1.22
+++ Packet.py 17 Jan 2003 06:18:06 -0000 1.23
@@ -28,7 +28,7 @@
from mixminion.Common import MixError, floorDiv, isSMTPMailbox
# Major and minor number for the understood packet format.
-MAJOR_NO, MINOR_NO = 0,1
+MAJOR_NO, MINOR_NO = 0,1 #XXXX003 Bump minor_no for 0.0.3
# Length of a Mixminion message
MESSAGE_LEN = 1 << 15
Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- ServerInfo.py 6 Jan 2003 07:03:24 -0000 1.36
+++ ServerInfo.py 17 Jan 2003 06:18:06 -0000 1.37
@@ -43,6 +43,7 @@
"""A ServerInfo object holds a parsed server descriptor."""
_restrictFormat = 1
+ _restrictKeys = 0
_syntax = {
"Server" : { "__SECTION__": ("REQUIRE", None, None),
"Descriptor-Version": ("REQUIRE", None, None),
@@ -57,6 +58,8 @@
"Contact": ("ALLOW", None, None),
"Comments": ("ALLOW", None, None),
"Packet-Key": ("REQUIRE", C._parsePublicKey, None),
+ "Packet-Formats": ("ALLOW", None, None),
+ "Software": ("ALLOW", None, None),
},
"Incoming/MMTP" : {
"Version": ("REQUIRE", None, None),
@@ -335,12 +338,14 @@
# expectedDigest: the 20-byte digest we expect to find in this
# directory's header.
_restrictFormat = 1
+ _restrictKeys = 0
_syntax = {
'Directory': { "__SECTION__": ("REQUIRE", None, None),
"Version": ("REQUIRE", None, None),
"Published": ("REQUIRE", C._parseTime, None),
"Valid-After": ("REQUIRE", C._parseDate, None),
"Valid-Until": ("REQUIRE", C._parseDate, None),
+ "Recommended-Software": (None, None, None),
},
'Signature': {"__SECTION__": ("REQUIRE", None, None),
"DirectoryIdentity": ("REQUIRE", C._parsePublicKey, None),
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.74
retrieving revision 1.75
diff -u -d -r1.74 -r1.75
--- test.py 14 Jan 2003 09:20:17 -0000 1.74
+++ test.py 17 Jan 2003 06:18:06 -0000 1.75
@@ -2523,6 +2523,25 @@
self.assertEquals(readFile(t).count("\n") , 1)
self.assertEquals(readFile(t1).count("\n"), 3)
+ def testLogStream(self):
+ stream = mixminion.Common.LogStream("STREAM", "WARN")
+ suspendLog()
+ try:
+ print >>stream, "Testing", 1,2,3
+ print >>stream
+ print >>stream, "A\nB\nC"
+ print >>stream, "X",
+ print >>stream, "Y"
+ finally:
+ r = resumeLog()
+ lines = [ l[20:] for l in r.split("\n") ]
+ self.assertEquals(lines[0], "[WARN] ->STREAM: Testing 1 2 3")
+ self.assertEquals(lines[1], "[WARN] ->STREAM: ")
+ self.assertEquals(lines[2], "[WARN] ->STREAM: A")
+ self.assertEquals(lines[3], "[WARN] ->STREAM: B")
+ self.assertEquals(lines[4], "[WARN] ->STREAM: C")
+ self.assertEquals(lines[5], "[WARN] ->STREAM: X Y")
+
#----------------------------------------------------------------------
# File paranoia
@@ -2704,7 +2723,6 @@
return server, listener, messagesIn, keyid
class MMTPTests(unittest.TestCase):
-
def doTest(self, fn):
"""Wraps an underlying test function 'fn' to make sure we kill the
MMTPServer, but don't block."""
@@ -2765,6 +2783,19 @@
server.process(0.1)
t.join()
+ def testStallingTransmission(self):
+ now = time.time()
+ try:
+ mixminion.MMTPClient.sendMessages("0.0.0.1",
+ #Is there a better IP????
+ TEST_PORT, "Z"*20, ["JUNK"],
+ connectTimeout=1)
+ self.fail("Expected the connection to time out")
+ except mixminion.MMTPClient.TimeoutError:
+ pass
+ passed = time.time() - now
+ self.assert_(passed < 2)
+
def _testNonblockingTransmission(self):
server, listener, messagesIn, keyid = _getMMTPServer()
self.listener = listener
@@ -3092,7 +3123,7 @@
# Command
if not sys.platform == 'win32':
- # FFFF This should get implemented for Windows.
+ # WIN32 This should get implemented for Windows.
self.assertEquals(C._parseCommand("ls -l"), ("/bin/ls", ['-l']))
self.assertEquals(C._parseCommand("rm"), ("/bin/rm", []))
self.assertEquals(C._parseCommand("/bin/ls"), ("/bin/ls", []))
@@ -3190,6 +3221,24 @@
# This is what we expect
pass
+ # IntervalSet validation
+ def warns(mixInterval, retryList, self=self):
+ ents = { "Section":
+ [('Retry', mixminion.Config._parseIntervalList(retryList))]}
+ try:
+ suspendLog()
+ mixminion.server.ServerConfig._validateRetrySchedule(
+ mixInterval, ents, "Section")
+ finally:
+ r = resumeLog()
+ self.assert_(stringContains(r, "[WARN]"))
+ warns(30*60, "every .6 hour for 20 hours") # < 1day
+ warns(30*60, "every 4 days for 1 month") # > 2 weeks
+ warns(30*60, "every 2 days for 4 days") # < twice
+ warns(30*60, "every .2 hours for 1 hour, every 1 day for 1 week")#<mix
+ warns(30*60, "every 5 days for 1 week") # too few attempts
+ warns(30*60, "every 1 hour for 1 week") # too many attempts
+
# Fractions
fails(SC._parseFraction, "90")
fails(SC._parseFraction, "-.01")
@@ -3224,9 +3273,11 @@
[Outgoing/MMTP]
Enabled = yes
Allow: *
+Retry: every 1 hour for 1 day, every 1 day for 1 week
[Delivery/MBOX]
Enabled: no
+Retry: every 1 hour for 1 day, every 1 day for 1 week
"""
@@ -3335,6 +3386,11 @@
inf2 = inf2.replace("0.1\n", "0.1 \n")
mixminion.ServerInfo.ServerInfo(string=inf2)
+ # Make sure we accept an extra key.
+ inf2 = inf+"Unexpected-Key: foo\n"
+ inf2 = mixminion.ServerInfo.signServerInfo(inf2, identity)
+ mixminion.ServerInfo.ServerInfo(string=inf2)
+
# Now make sure everything was saved properly
keydir = os.path.join(d, "key_key1")
eq(inf, readFile(os.path.join(keydir, "ServerDesc")))
@@ -4075,7 +4131,7 @@
"AddressFile": addrfile,
"ReturnAddress": "returnaddress@x",
"RemoveContact": "removeaddress@x",
- "Retry": [0,0,0,0],
+ "Retry": [0,0,0,3],
"SMTPServer" : "foo.bar.baz"}}, manager)
# Check that the address file was read correctly.
self.assertEquals({'mix-minion': 'mixminion@thishost',
@@ -4086,6 +4142,7 @@
# Stub out sendSMTPMessage.
replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
lambda *args: mixminion.server.Modules.DELIVER_OK)
+ self.assertEquals(queue.retrySchedule, [0,0,0,3])
try:
# Try queueing a message...
queue.queueDeliveryMessage(FDP('enc', MBOX_TYPE, 'mixdiddy',
@@ -4415,11 +4472,13 @@
addrf = mix_mktemp()
writeFile(addrf,"")
conf += ("[Delivery/MBOX]\nEnabled: yes\nAddressFile: %s\n"+
- "ReturnAddress: a@b.c\nRemoveContact: b@c.d\n") %(
+ "ReturnAddress: a@b.c\nRemoveContact: b@c.d\n"+
+ "Retry: every 2 hours for 1 week\n") %(
addrf)
elif t == SMTP_TYPE:
conf += ("[Delivery/SMTP]\nEnabled: yes\n"+
- "ReturnAddress: a@b.c\n")
+ "ReturnAddress: a@b.c\n"+
+ "Retry: every 2 hours for 1 week\n")
else:
raise MixFatalError("Unrecognized type: %04x"%t)
try:
@@ -5050,8 +5109,9 @@
self.keyid = keyid
self.packets = []
self.connected = 0
- def connect(self):
+ def connect(self, connectTimeout):
self.connected = 1
+ self.timeout = connectTimeout
def sendPacket(self, msg):
assert self.connected
self.packets.append(msg)
@@ -5103,7 +5163,7 @@
tc = loader.loadTestsFromTestCase
if 0:
- suite.addTest(tc(BuildMessageTests))
+ suite.addTest(tc(MMTPTests))
return suite
suite.addTest(tc(MiscTests))