[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[minion-cvs] Tweaks and fixes in response to integration tests.



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv25273/lib/mixminion

Modified Files:
	ClientMain.py Common.py Main.py ServerInfo.py test.py 
Log Message:
Tweaks and fixes in response to integration tests.

ClientMain:
	- Give an actual URL and fingerprint for our (ersatz but operational) 
	  directory server.
	- Rename "servers" directory to "imported"; if we see any directories
	  named "servers" sitting around, gripe and try to remove them.
	- Check for expired or superseded descriptors on import.
	- Simplify "clean" logic a lot.
	- Fiddle with log messages and user-visible messages; add more warnings
	- Make the default path length configurable

ClientMain, Main:
	- Add 'import-server' command.

Common:
	- Only warn about fishy directory permissions once per directory.
	- Normalize paths before checking their permissions.
	- Use gzip a bit more pedantically.

test:
	- Remove a dead test

DirMain:
	- Use gzip a bit more pedantically.

ServerConfig, ServerMain:
	- Rename the confusing 'NoDaemon' option to 'Daemon' and flip its
	  semantics.

ServerConfig
	- Change the default MixAlgorithm from "Cottrell" to "Timed".




Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- ClientMain.py	5 Jan 2003 01:27:35 -0000	1.22
+++ ClientMain.py	5 Jan 2003 04:29:11 -0000	1.23
@@ -40,10 +40,9 @@
 from mixminion.Packet import ParseError, parseMBOXInfo, parseSMTPInfo, \
      MBOX_TYPE, SMTP_TYPE, DROP_TYPE
 
-# FFFF This should be made configurable.
+# FFFF This should be made configurable and adjustable.
 MIXMINION_DIRECTORY_URL = "http://www.mixminion.net/directory/latest.gz";
-# FFFF This should be made configurable.
-MIXMINION_DIRECTORY_FINGERPRINT = ""
+MIXMINION_DIRECTORY_FINGERPRINT = "CD80DD1B8BE7CA2E13C928D57499992D56579CCD"
 
 class ClientKeystore:
     """A ClientKeystore manages a list of server descriptors, either
@@ -65,10 +64,10 @@
     # DIR/cache: A cPickled tuple of ("ClientKeystore-0",
     #         lastModified, lastDownload, serverlist, digestMap)
     # DIR/dir.gz *or* DIR/dir: A (possibly gzipped) directory file.
-    # DIR/servers/: A directory of server descriptors.
+    # DIR/imported/: A directory of server descriptors.
 
     MAGIC = "ClientKeystore-0"
-    #DOCDOC
+    # 
     DEFAULT_REQUIRED_LIFETIME = 3600
     
     def __init__(self, directory):
@@ -76,9 +75,27 @@
            under <directory>."""
         self.dir = directory
         createPrivateDir(self.dir)
+        createPrivateDir(os.path.join(self.dir, "imported"))
         self.digestMap = {}
         self.__scanning = 0
         self.__load()
+        self.clean()
+
+        # Mixminion 0.0.1 used an obsolete directory-full-of-servers in 
+        #   DIR/servers.  If there's nothing there, we remove it.  Otherwise,
+        #   we warn.
+        sdir = os.path.join(self.dir,"servers")
+        if os.path.exists(sdir):
+            if os.listdir(sdir):
+                LOG.warn("Skipping obsolete server directory %s", sdir)
+            else:
+                try:
+                    LOG.warn("Removing obsolete server directory %s", sdir)
+                    os.rmdir(sdir)
+                    print >>sys.stderr, "OK"
+                except OSError, e:
+                    print >>sys.stderr, "BAD"
+                    LOG.warn("Failed: %s", e)
 
     def updateDirectory(self, forceDownload=0, now=None):
         """Download a directory from the network as needed."""
@@ -171,7 +188,7 @@
             break
 
         # Now check the server in DIR/servers.
-        serverDir = os.path.join(self.dir, "servers")
+        serverDir = os.path.join(self.dir, "imported")
         createPrivateDir(serverDir)
         for fn in os.listdir(serverDir):
             # Try to read a file: is it a server descriptor?
@@ -253,10 +270,19 @@
         # Have we already imported this server?
         if self.digestMap.get(info.getDigest(), "X").startswith("I:"):
             raise MixError("Server descriptor is already imported")
+
+        # Is the server expired?
+        if info.isExpiredAt(time.time()):
+            raise MixError("Server desciptor is expired")
+
+        # Is the server superseded?
+        if self.byNickname.has_key(nickname):
+            if info.isSupersededBy([s for s, _ in self.byNickname[nickname]]):
+                raise MixError("Server descriptor is superseded")
         
         # Copy the server into DIR/servers.
         fnshort = "%s-%s"%(nickname, formatFnameTime())
-        fname = os.path.join(self.dir, "servers", fnshort)
+        fname = os.path.join(self.dir, "imported", fnshort)
         f = openUnique(fname)[0]
         f.write(contents)
         f.close()
@@ -280,7 +306,7 @@
             n += 1
             try:
                 fn = source[2:]
-                os.unlink(os.path.join(self.dir, "servers", fn))
+                os.unlink(os.path.join(self.dir, "imported", fn))
             except OSError, e:
                 LOG.error("Couldn't remove %s: %s", fn, e)
 
@@ -376,27 +402,18 @@
 
         newServers = []
         for info, where in self.serverList:
-            if where == 'D':
-                # Don't scratch servers from directory.
-                newServers.append((info, where))
-                continue
-            elif info.isExpiredAt(cutoff):
-                pass
+            others = [ s for s, _ in self.byNickname[info.getNickname()] ]
+            if (where != 'D'
+                and (info.isExpiredAt(cutoff)
+                     or info.isSupersededBy(others))):
+                # Otherwise, remove it.
+                try:
+                    os.unlink(os.path.join(self.dir, "imported", where[2:]))
+                except OSError, e:
+                    LOG.info("Couldn't remove %s: %s", where[2:], e)
             else:
-                valid = info.getIntervalSet()
-                for s, _ in self.byNickname[info.getNickname()]:
-                    if s.isNewerThan(info):
-                        valid -= s.getIntervalSet()
-                if not valid.isEmpty():
-                    # Don't scratch non-superseded, non-expired servers.
-                    newServers.append((info, where))
-                    continue
-            
-            # Otherwise, remove it.
-            try:
-                os.unlink(os.path.join(self.dir, "servers", where[2:]))
-            except OSError, e:
-                LOG.info("Couldn't remove %s: %s", where[2:], e)
+                # Don't scratch non-superseded, non-expired servers.
+                newServers.append((info, where))            
 
         if len(self.serverList) != len(newServers):
             self.serverList = newServers
@@ -558,10 +575,10 @@
             # We don't know any servers at all.
             raise MixError("No relays known")
 
-        LOG.info("Chose path: [%s][%s][%s]",
-                 " ".join([ s.getNickname() for s in startServers ]),
-                 " ".join([ s.getNickname() for s in midServers   ]),
-                 " ".join([ s.getNickname() for s in endServers   ]))
+        LOG.debug("getPath: [%s][%s][%s]",
+                  " ".join([ s.getNickname() for s in startServers ]),
+                  " ".join([ s.getNickname() for s in midServers   ]),
+                  " ".join([ s.getNickname() for s in endServers   ]))
 
         return startServers + midServers + endServers
 
@@ -607,11 +624,15 @@
                            % server.getNickname())
     if exitCap and exitCap not in path[-1].getCaps():
         raise MixError("Server %s does not support %s"
-                       % (server.getNickname(), exitCap))
+                       % (path[-1].getNickname(), exitCap))
  
     if nSwap is None:
         nSwap = ceilDiv(len(path),2)-1
-    return path[:nSwap+1], path[nSwap+1:]
+
+    path1, path2 = path[:nSwap+1], path[nSwap+1:]
+    if not path1 or not path2:
+        raise MixError("Each leg of the path must have at least 1 hop")
+    return path1, path2
 
 def parsePath(keystore, config, path, address, nHops=None, 
               nSwap=None, startAt=None, endAt=None):
@@ -682,7 +703,13 @@
     if starPos is None:
         myNHops = len(enterPath)
     else:
-        myNHops = nHops or 6 # FFFF Configurable default!
+        if nHops:
+            myNHops = nHops
+        elif config is not None:
+            myNHops = config['Security'].get("PathLength", 6)
+        else:
+            myNHops = 6
+            
 
     if swapPos is None:
         # a,b,c,d or a,b,*,c
@@ -767,8 +794,6 @@
         # Make directories
         userdir = os.path.expanduser(self.config['User']['UserDir'])
         createPrivateDir(userdir)
-        #createPrivateDir(os.path.join(userdir, 'surbs'))
-        createPrivateDir(os.path.join(userdir, 'servers'))
 
         # Initialize PRNG
         self.prng = mixminion.Crypto.getCommonPRNG()
@@ -944,11 +969,13 @@
         elif opt in ('-H', '--hops'):
             try:
                 nHops = int(val)
+                if nHops < 2:
+                    usageAndExit(cmd, "Must have at least 2 hops")
             except ValueError:
                 usageAndExit(cmd, "%s expects an integer"%opt)
         elif opt == '--swap-at':
             try:
-                nHops = int(val)
+                nSwap = int(val)-1
             except ValueError:
                 usageAndExit(cmd, "%s expects an integer"%opt)
         elif opt in ('-t', '--to'):
@@ -961,6 +988,7 @@
                 download = 1
             else:
                 usageAndExit(cmd, "Unrecognized value for %s"%opt)
+
     if args:
         usageAndExit(cmd,"Unexpected options")
     if address is None:
@@ -969,7 +997,9 @@
     config = readConfigFile(configFile)
     LOG.configure(config)
     if verbose:
-        LOG.setMinSeverity("DEBUG")
+        LOG.setMinSeverity("TRACE")
+    else:
+        LOG.setMinSeverity("INFO")
 
     LOG.debug("Configuring client")
     mixminion.Common.configureShredCommand(config)
@@ -979,12 +1009,14 @@
     if download != 0:
         keystore.updateDirectory(forceDownload=download)
     
-    #try:
-    if 1:
+    try:
         path1, path2 = parsePath(keystore, config, path, address, nHops, nSwap)
-    #except MixError, e:
-    #    print e
-    #    sys.exit(1)
+        LOG.info("Chose path: [%s][%s]",
+                 " ".join([ s.getNickname() for s in path1 ]),
+                 " ".join([ s.getNickname() for s in path2 ]))
+    except MixError, e:
+        print >>sys.stderr, e
+        sys.exit(1)
 
     client = MixminionClient(config)
 
@@ -997,6 +1029,31 @@
 
     client.sendForwardMessage(address, payload, path1, path2)
 
+    print >>sys.stderr, "Message sent"
+
+def importServer(cmd, args):
+    options, args = getopt.getopt(args, "hf:", ['help', 'config='])
+    configFile = None
+    for o,v in options:
+        if o in ('-h', '--help'):
+            print "Usage %s [--help] [--config=configFile] <filename> ..."
+            sys.exit(1)
+        elif o in ('-f', '--config'):
+            configFile = v
+
+    config = readConfigFile(configFile)
+    userdir = os.path.expanduser(config['User']['UserDir'])
+    keystore = ClientKeystore(os.path.expanduser(config['User']['UserDir']))
+
+    for filename in args:
+        print "Importing from", filename
+        try:
+            keystore.importFromFile(filename)
+        except MixError, e:
+            print "Error while importing: %s" % e
+
+    print "Done."
+
 def listServers(cmd, args):
     options, args = getopt.getopt(args, "hf:", ['help', 'config='])
     configFile = None
@@ -1009,7 +1066,6 @@
 
     config = readConfigFile(configFile)
     userdir = os.path.expanduser(config['User']['UserDir'])
-    createPrivateDir(os.path.join(userdir, 'servers'))
 
     keystore = ClientKeystore(os.path.expanduser(config['User']['UserDir']))
         

Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -d -r1.43 -r1.44
--- Common.py	5 Jan 2003 01:28:11 -0000	1.43
+++ Common.py	5 Jan 2003 04:29:11 -0000	1.44
@@ -115,6 +115,8 @@
 
     checkPrivateDir(d)
 
+_WARNED_DIRECTORIES = {}
+
 def checkPrivateDir(d, recurse=1):
     """Return true iff d is a directory owned by this uid, set to mode
        0700. All of d's parents must not be writable or owned by anybody but
@@ -122,6 +124,9 @@
        MixFatalErrror.  Otherwise, return None."""
     me = os.getuid()
 
+    if not os.path.isabs(d):
+        d = os.path.abspath(d)
+
     if not os.path.exists(d):
         raise MixFatalError("Directory %s does not exist" % d)
     if not os.path.isdir(d):
@@ -156,9 +161,10 @@
 
         if (mode & 020) and not (mode & stat.S_ISVTX):
             # FFFF We may want to give an even stronger error here.
-            LOG.warn("Iffy mode %o on directory %s (Writable by gid %s)",
-                     mode, d, st[stat.ST_GID])
-
+            if not _WARNED_DIRECTORIES.has_key(d):
+                LOG.warn("Iffy mode %o on directory %s (Writable by gid %s)",
+                         mode, d, st[stat.ST_GID])
+            _WARNED_DIRECTORIES[d] = 1
 #----------------------------------------------------------------------
 # Secure filesystem operations.
 
@@ -385,9 +391,10 @@
                     self.addHandler(_FileLogHandler(logfile))
                 except MixError, e:
                     self.error(str(e))
-            if logfile and not (config['Server'].get('EchoMessages',0) and
-                                config['Server'].get('NoDaemon',0)):
-                del self.handlers[0]
+                if (config['Server'].get('Daemon',0) or
+                    not config['Server'].get('EchoMessages',0)):
+                    print "Removing console handler"
+                    del self.handlers[0]
 
     def setMinSeverity(self, minSeverity):
         """Sets the minimum severity of messages to be logged.
@@ -812,7 +819,7 @@
     f = None
     try:
         if fname.endswith(".gz"):
-            f = gzip.GzipFile(fname, 'r')
+            f = gzip.GzipFile(fname, 'rb')
         else:
             f = open(fname, 'r')
         return f.read()

Index: Main.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Main.py,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- Main.py	5 Jan 2003 01:28:27 -0000	1.17
+++ Main.py	5 Jan 2003 04:29:11 -0000	1.18
@@ -116,6 +116,7 @@
     "unittests" :      ( 'mixminion.test',       'testAll' ),
     "benchmarks" :     ( 'mixminion.benchmark',  'timeAll' ),
     "client" :         ( 'mixminion.ClientMain', 'runClient' ),
+    "import-server" :  ( 'mixminion.ClientMain', 'importServer' ),
     "list-servers" :   ( 'mixminion.ClientMain', 'listServers' ),
     "server" :         ( 'mixminion.server.ServerMain', 'runServer' ),
     "server-keygen" :  ( 'mixminion.server.ServerMain', 'runKeygen'),

Index: ServerInfo.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ServerInfo.py,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- ServerInfo.py	4 Jan 2003 04:12:51 -0000	1.31
+++ ServerInfo.py	5 Jan 2003 04:29:11 -0000	1.32
@@ -299,7 +299,7 @@
             contents = string
         else:
             contents = readPossiblyGzippedFile(fname)
-        
+
         contents = _cleanForDigest(contents)
 
         # First, get the digest.  Then we can break everything up.

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.59
retrieving revision 1.60
diff -u -d -r1.59 -r1.60
--- test.py	5 Jan 2003 01:29:55 -0000	1.59
+++ test.py	5 Jan 2003 04:29:11 -0000	1.60
@@ -4355,7 +4355,7 @@
             for idx in xrange(len(descriptors)):
                 fname = os.path.join(impdirname, "%s%s" % (server,idx))
                 writeFile(fname, descriptors[idx])
-                f = gzip.GzipFile(fname+".gz", 'w')
+                f = gzip.GzipFile(fname+".gz", 'wb')
                 f.write(descriptors[idx])
                 f.close()
 
@@ -4838,11 +4838,6 @@
         usercfgstr = "[User]\nUserDir: %s\n[DirectoryServers]\n"%userdir
         usercfg = mixminion.Config.ClientConfig(string=usercfgstr)
         client = mixminion.ClientMain.MixminionClient(usercfg)
-
-        # Make sure client sets its directories up correctly.
-        serverdir = os.path.join(userdir, 'servers')
-        self.assert_(os.path.exists(serverdir))
-        self.assertEquals([], os.listdir(serverdir))
 
         # Now try with some servers...
         edesc = getExampleServerDescriptors()