[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[minion-cvs] Timeouts, descriptor parsing, refactoring, more tests.



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv22424/lib/mixminion

Modified Files:
	ClientMain.py Common.py Config.py MMTPClient.py Packet.py 
	ServerInfo.py test.py 
Log Message:
Timeouts, descriptor parsing, refactoring, more tests.

ClientMain, MMTPClient, Config:
- Support configurable client timeout

Common:
- Make stdout/stderr wrappers not suck

Config, ServerInfo:
- Add support for ServerInfo and directory parsing to ignore unrecognized
  keys

Packet, ServerList, ..., ServerKeys:
- Note work for 0.0.3

ServerMain:
- Change ProcessingThread to be callable-based, not packet-based.

test:
- Tests for log streams
- Tests for client timeouts
- Tests for interval list warnings



Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -d -r1.43 -r1.44
--- ClientMain.py	13 Jan 2003 06:13:33 -0000	1.43
+++ ClientMain.py	17 Jan 2003 06:18:06 -0000	1.44
@@ -82,6 +82,7 @@
         self.__scanning = 0
         self.__load()
         self.clean()
+        #XXXX003 Check version against directory's Recommended-Software field.
 
         # Mixminion 0.0.1 used an obsolete directory-full-of-servers in
         #   DIR/servers.  If there's nothing there, we remove it.  Otherwise,
@@ -789,6 +790,9 @@
 [Security]
 PathLength: 4
 
+[Network]
+ConnectionTimeout: 20 seconds
+
 """)
     f.close()
 
@@ -844,19 +848,19 @@
         """Given a list of packets and a ServerInfo object, sends the
            packets to the server via MMTP"""
         LOG.info("Connecting...")
-        con = mixminion.MMTPClient.BlockingClientConnection(server.getAddr(),
-                                                            server.getPort(),
-                                                            server.getKeyID())
+        timeout = self.config['Network'].get('ConnectionTimeout')
+        if timeout:
+            timeout = timeout[2]
+
         try:
-            try:
-                con.connect()
-                LOG.info("Sending packet(s)")
-                for msg in msgList:
-                    con.sendPacket(msg)
-            except socket.error, e:
-                raise MixError("Error sending packets: %s" % e)
-        finally:
-            con.shutdown()
+            # May raise TimeoutError
+            mixminion.MMTPClient.sendMessages(server.getAddr(),
+                                              server.getPort(),
+                                              server.getKeyID(),
+                                              msgList,
+                                              timeout)
+        except socket.error, e:
+            raise MixError("Error sending packets: %s" % e)
 
 def parseAddress(s):
     """Parse and validate an address; takes a string, and returns an Address
@@ -1077,7 +1081,7 @@
         keystore.updateDirectory(forceDownload=download)
 
     if address is None:
-        print "No recipients specified; exiting."
+        print >>sys.stderr, "No recipients specified; exiting."
         sys.exit(0)
 
     try:

Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.52
retrieving revision 1.53
diff -u -d -r1.52 -r1.53
--- Common.py	10 Jan 2003 20:12:05 -0000	1.52
+++ Common.py	17 Jan 2003 06:18:06 -0000	1.53
@@ -534,8 +534,24 @@
     def __init__(self, name, severity):
         self.name = name
         self.severity = severity
+        self.buf = [] #DOCDOC
     def write(self, s):
-        LOG.log(self.severity, "->%s: %s", self.name, s)
+        #DOCDOC justify this inefficiency.
+        #LOG.log(self.severity, "called with %r", s)
+        if "\n" not in s:
+            self.buf.append(s)
+            return
+
+        while "\n" in s:
+            idx = s.index("\n")
+            line = "%s%s" %("".join(self.buf), s[:idx])
+            LOG.log(self.severity, "->%s: %s", self.name, line)
+            del self.buf[:]
+            s = s[idx+1:]
+            
+        if s:
+            self.buf.append(s)
+                            
     def flush(self): pass
     def close(self): pass
 
@@ -767,7 +783,7 @@
         options = os.WNOHANG
     while 1:
         try:
-            # FFFF This won't work on Windows.  What to do?
+            # WIN32 This won't work on Windows.  What to do?
             pid, status = os.waitpid(0, options)
         except OSError, e:
             break
@@ -784,7 +800,7 @@
 
     while 1:
         try:
-            # This waitpid call won't work on Windows.  What to do?
+            # WIN32 This waitpid call won't work on Windows.  What to do?
             pid, status = os.waitpid(0, os.WNOHANG)
             if pid == 0:
                 break

Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.35
retrieving revision 1.36
diff -u -d -r1.35 -r1.36
--- Config.py	13 Jan 2003 06:15:10 -0000	1.35
+++ Config.py	17 Jan 2003 06:18:06 -0000	1.36
@@ -497,6 +497,7 @@
 
     _syntax = None
     _restrictFormat = 0
+    _restrictKeys = 1 #DOCDOC
 
     def __init__(self, filename=None, string=None, assumeValid=0):
         """Create a new _ConfigFile.  If <filename> is set, read from
@@ -582,8 +583,12 @@
                 try:
                     rule, parseFn, default = secConfig[k]
                 except KeyError:
-                    raise ConfigError("Unrecognized key %s on line %s" %
-                                      (k, line))
+                    if self._restrictKeys:
+                        raise ConfigError("Unrecognized key %s on line %s" %
+                                          (k, line))
+                    else:
+                        LOG.warn("Unregognized key %s on line %s", k, line)
+                        continue
 
                 # Parse and validate the value of this entry.
                 if parseFn is not None:
@@ -711,6 +716,7 @@
                        'SURBAddress' : ('ALLOW', None, None),
                        'SURBPathLength' : ('ALLOW', _parseInt, "8"),
                        'SURBLifetime' : ('ALLOW', _parseInterval, "7 days") },
+        'Network' : { 'ConnectionTimeout' : ('ALLOW', _parseInterval, None) }
         }
     def __init__(self, fname=None, string=None):
         _ConfigFile.__init__(self, fname, string)
@@ -725,6 +731,13 @@
         if p < 4:
             LOG.warn("Your default path length is frighteningly low."
                           "  I'll trust that you know what you're doing.")
+
+        t = sections['Network'].get('ConnectionTimeout')
+        if t:
+            if t[2] < 5:
+                LOG.warn("Very short connection timeout")
+            elif t[2] > 60:
+                LOG.warn("Very long connection timeout")
 
 def _validateHostSection(sec):
     """Helper function: Makes sure that the shared [Host] section is correct;

Index: MMTPClient.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/MMTPClient.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- MMTPClient.py	14 Jan 2003 09:20:17 -0000	1.17
+++ MMTPClient.py	17 Jan 2003 06:18:06 -0000	1.18
@@ -53,11 +53,11 @@
         self.tls = None
         self.sock = None
 
-    def connect(self, timeout=None):
+    def connect(self, connectTimeout=None):
         """Negotiate the handshake and protocol."""
         def sigalarmHandler(sig, _):
             assert sig == signal.SIGALRM
-        if timeout:
+        if connectTimeout:
             signal.signal(signal.SIGALRM, sigalarmHandler)
         
         # Connect to the server
@@ -66,15 +66,20 @@
         LOG.debug("Connecting to %s:%s", self.targetIP, self.targetPort)
 
         # Do the TLS handshaking
-        if timeout:
-            signal.alarm(timeout)
+        if connectTimeout:
+            signal.alarm(connectTimeout)
         try:
-            self.sock.connect((self.targetIP,self.targetPort))
-        except socket.error, e:
-            if e[0] == errno.EINTR:
-                raise TimeoutError("Connection timed out")
-            else:
-                raise e
+            try:
+                self.sock.connect((self.targetIP,self.targetPort))
+            except socket.error, e:
+                if e[0] == errno.EINTR:
+                    raise TimeoutError("Connection timed out")
+                else:
+                    raise e
+        finally:
+            if connectTimeout:
+                signal.alarm(0)
+            
         LOG.debug("Handshaking with %s:%s",self.targetIP, self.targetPort)
         self.tls = self.context.sock(self.sock.fileno())
         # FFFF session resumption
@@ -152,7 +157,8 @@
             self.sock.close()
         LOG.debug("Connection closed")
 
-def sendMessages(targetIP, targetPort, targetKeyID, packetList):
+def sendMessages(targetIP, targetPort, targetKeyID, packetList,
+                 connectTimeout=None):
     """Sends a list of messages to a server.
         DOCDOC arguments
         DOCDOC "JUNK", "RENEGOTIATE"
@@ -169,7 +175,7 @@
     
     con = BlockingClientConnection(targetIP, targetPort, targetKeyID)
     try:
-        con.connect()
+        con.connect(connectTimeout=connectTimeout)
         for t,p in packets:
             if t == "JUNK":
                 con.sendJunkPacket(p)

Index: Packet.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Packet.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- Packet.py	16 Dec 2002 02:40:11 -0000	1.22
+++ Packet.py	17 Jan 2003 06:18:06 -0000	1.23
@@ -28,7 +28,7 @@
 from mixminion.Common import MixError, floorDiv, isSMTPMailbox
 
 # Major and minor number for the understood packet format.
-MAJOR_NO, MINOR_NO = 0,1
+MAJOR_NO, MINOR_NO = 0,1  #XXXX003 Bump minor_no for 0.0.3
 
 # Length of a Mixminion message
 MESSAGE_LEN = 1 << 15

Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- ServerInfo.py	6 Jan 2003 07:03:24 -0000	1.36
+++ ServerInfo.py	17 Jan 2003 06:18:06 -0000	1.37
@@ -43,6 +43,7 @@
 
     """A ServerInfo object holds a parsed server descriptor."""
     _restrictFormat = 1
+    _restrictKeys = 0
     _syntax = {
         "Server" : { "__SECTION__": ("REQUIRE", None, None),
                      "Descriptor-Version": ("REQUIRE", None, None),
@@ -57,6 +58,8 @@
                      "Contact": ("ALLOW", None, None),
                      "Comments": ("ALLOW", None, None),
                      "Packet-Key": ("REQUIRE", C._parsePublicKey, None),
+                     "Packet-Formats": ("ALLOW", None, None),
+                     "Software": ("ALLOW", None, None),
                      },
         "Incoming/MMTP" : {
                      "Version": ("REQUIRE", None, None),
@@ -335,12 +338,14 @@
     # expectedDigest: the 20-byte digest we expect to find in this
     #    directory's header.
     _restrictFormat = 1
+    _restrictKeys = 0
     _syntax = {
         'Directory': { "__SECTION__": ("REQUIRE", None, None),
                        "Version": ("REQUIRE", None, None),
                        "Published": ("REQUIRE", C._parseTime, None),
                        "Valid-After": ("REQUIRE", C._parseDate, None),
                        "Valid-Until": ("REQUIRE", C._parseDate, None),
+                       "Recommended-Software": (None, None, None),
                        },
         'Signature': {"__SECTION__": ("REQUIRE", None, None),
                  "DirectoryIdentity": ("REQUIRE", C._parsePublicKey, None),

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.74
retrieving revision 1.75
diff -u -d -r1.74 -r1.75
--- test.py	14 Jan 2003 09:20:17 -0000	1.74
+++ test.py	17 Jan 2003 06:18:06 -0000	1.75
@@ -2523,6 +2523,25 @@
         self.assertEquals(readFile(t).count("\n") , 1)
         self.assertEquals(readFile(t1).count("\n"), 3)
 
+    def testLogStream(self):
+        stream = mixminion.Common.LogStream("STREAM", "WARN")
+        suspendLog()
+        try:
+            print >>stream, "Testing", 1,2,3
+            print >>stream
+            print >>stream, "A\nB\nC"
+            print >>stream, "X",
+            print >>stream, "Y"
+        finally:
+            r = resumeLog()
+        lines = [ l[20:] for l in r.split("\n") ]
+        self.assertEquals(lines[0], "[WARN] ->STREAM: Testing 1 2 3")
+        self.assertEquals(lines[1], "[WARN] ->STREAM: ")
+        self.assertEquals(lines[2], "[WARN] ->STREAM: A")
+        self.assertEquals(lines[3], "[WARN] ->STREAM: B")
+        self.assertEquals(lines[4], "[WARN] ->STREAM: C")
+        self.assertEquals(lines[5], "[WARN] ->STREAM: X Y")
+
 #----------------------------------------------------------------------
 # File paranoia
 
@@ -2704,7 +2723,6 @@
     return server, listener, messagesIn, keyid
 
 class MMTPTests(unittest.TestCase):
-
     def doTest(self, fn):
         """Wraps an underlying test function 'fn' to make sure we kill the
            MMTPServer, but don't block."""
@@ -2765,6 +2783,19 @@
             server.process(0.1)
         t.join()
 
+    def testStallingTransmission(self):
+        now = time.time()
+        try:
+            mixminion.MMTPClient.sendMessages("0.0.0.1",
+                                              #Is there a better IP????
+                                              TEST_PORT, "Z"*20, ["JUNK"],
+                                              connectTimeout=1)
+            self.fail("Expected the connection to time out")
+        except mixminion.MMTPClient.TimeoutError:
+            pass
+        passed = time.time() - now
+        self.assert_(passed < 2)
+
     def _testNonblockingTransmission(self):
         server, listener, messagesIn, keyid = _getMMTPServer()
         self.listener = listener
@@ -3092,7 +3123,7 @@
 
         # Command
         if not sys.platform == 'win32':
-            # FFFF This should get implemented for Windows.
+            # WIN32 This should get implemented for Windows.
             self.assertEquals(C._parseCommand("ls -l"), ("/bin/ls", ['-l']))
             self.assertEquals(C._parseCommand("rm"), ("/bin/rm", []))
             self.assertEquals(C._parseCommand("/bin/ls"), ("/bin/ls", []))
@@ -3190,6 +3221,24 @@
             # This is what we expect
             pass
 
+        # IntervalSet validation
+        def warns(mixInterval, retryList, self=self):
+            ents = { "Section":
+               [('Retry', mixminion.Config._parseIntervalList(retryList))]}
+            try:
+                suspendLog()
+                mixminion.server.ServerConfig._validateRetrySchedule(
+                    mixInterval, ents, "Section")
+            finally:
+                r = resumeLog()
+            self.assert_(stringContains(r, "[WARN]"))
+        warns(30*60, "every .6 hour for 20 hours") # < 1day
+        warns(30*60, "every 4 days for 1 month") # > 2 weeks
+        warns(30*60, "every 2 days for 4 days") # < twice
+        warns(30*60, "every .2 hours for 1 hour, every 1 day for 1 week")#<mix
+        warns(30*60, "every 5 days for 1 week") # too few attempts
+        warns(30*60, "every 1 hour for 1 week") # too many attempts
+
         # Fractions
         fails(SC._parseFraction, "90")
         fails(SC._parseFraction, "-.01")
@@ -3224,9 +3273,11 @@
 [Outgoing/MMTP]
 Enabled = yes
 Allow: *
+Retry: every 1 hour for 1 day, every 1 day for 1 week
 
 [Delivery/MBOX]
 Enabled: no
+Retry: every 1 hour for 1 day, every 1 day for 1 week
 
 """
 
@@ -3335,6 +3386,11 @@
         inf2 = inf2.replace("0.1\n", "0.1  \n")
         mixminion.ServerInfo.ServerInfo(string=inf2)
 
+        # Make sure we accept an extra key.
+        inf2 = inf+"Unexpected-Key: foo\n"
+        inf2 = mixminion.ServerInfo.signServerInfo(inf2, identity)
+        mixminion.ServerInfo.ServerInfo(string=inf2)
+
         # Now make sure everything was saved properly
         keydir = os.path.join(d, "key_key1")
         eq(inf, readFile(os.path.join(keydir, "ServerDesc")))
@@ -4075,7 +4131,7 @@
                            "AddressFile": addrfile,
                            "ReturnAddress": "returnaddress@x",
                            "RemoveContact": "removeaddress@x",
-                           "Retry": [0,0,0,0],
+                           "Retry": [0,0,0,3],
                            "SMTPServer" : "foo.bar.baz"}}, manager)
         # Check that the address file was read correctly.
         self.assertEquals({'mix-minion': 'mixminion@thishost',
@@ -4086,6 +4142,7 @@
         # Stub out sendSMTPMessage.
         replaceFunction(mixminion.server.Modules, 'sendSMTPMessage',
                  lambda *args: mixminion.server.Modules.DELIVER_OK)
+        self.assertEquals(queue.retrySchedule, [0,0,0,3])
         try:
             # Try queueing a message...
             queue.queueDeliveryMessage(FDP('enc', MBOX_TYPE, 'mixdiddy', 
@@ -4415,11 +4472,13 @@
                 addrf = mix_mktemp()
                 writeFile(addrf,"")
                 conf += ("[Delivery/MBOX]\nEnabled: yes\nAddressFile: %s\n"+
-                         "ReturnAddress: a@b.c\nRemoveContact: b@c.d\n") %(
+                         "ReturnAddress: a@b.c\nRemoveContact: b@c.d\n"+
+                         "Retry: every 2 hours for 1 week\n") %(
                     addrf)
             elif t == SMTP_TYPE:
                 conf += ("[Delivery/SMTP]\nEnabled: yes\n"+
-                         "ReturnAddress: a@b.c\n")
+                         "ReturnAddress: a@b.c\n"+
+                         "Retry: every 2 hours for 1 week\n")
             else:
                 raise MixFatalError("Unrecognized type: %04x"%t)
         try:
@@ -5050,8 +5109,9 @@
                 self.keyid = keyid
                 self.packets = []
                 self.connected = 0
-            def connect(self):
+            def connect(self, connectTimeout):
                 self.connected = 1
+                self.timeout = connectTimeout
             def sendPacket(self, msg):
                 assert self.connected
                 self.packets.append(msg)
@@ -5103,7 +5163,7 @@
     tc = loader.loadTestsFromTestCase
 
     if 0:
-        suite.addTest(tc(BuildMessageTests))
+        suite.addTest(tc(MMTPTests))
         return suite
 
     suite.addTest(tc(MiscTests))