[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[minion-cvs] Debug refactored ClientDirectory. Now it will be easie...



Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv8278/lib/mixminion

Modified Files:
	ClientDirectory.py ClientMain.py Common.py Config.py 
	ThreadUtils.py test.py 
Log Message:
Debug refactored ClientDirectory.  Now it will be easier to use it for pinging.

Index: ClientDirectory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientDirectory.py,v
retrieving revision 1.38
retrieving revision 1.39
diff -u -d -r1.38 -r1.39
--- ClientDirectory.py	26 Apr 2004 16:55:45 -0000	1.38
+++ ClientDirectory.py	2 May 2004 18:45:15 -0000	1.39
@@ -9,7 +9,7 @@
 """
 
 __all__ = [ 'ClientDirectory', 'parsePath', 'parseAddress',
-            'GotInvalidDirectoryError' ]
+            'DirectoryDownloadError', 'GotInvalidDirectoryError' ]
 
 import cPickle
 import errno
@@ -35,17 +35,17 @@
 from mixminion.Packet import MBOX_TYPE, SMTP_TYPE, DROP_TYPE, FRAGMENT_TYPE, \
      parseMBOXInfo, parseRelayInfoByType, parseSMTPInfo, ParseError, \
[...2342 lines suppressed...]
@@ -2793,7 +1712,8 @@
         return self.getMinLength()
 
 class ServerPathElement(PathElement):
-    """A path element for a single server specified by filename or nickname"""
+    """A path element for a single server specified by filename or nickname
+    """
     def __init__(self, nickname):
         self.nickname = nickname
     def validate(self, directory, start, end):
@@ -2832,7 +1752,8 @@
 class RandomServersPathElement(PathElement):
     """A path element for randomly chosen servers.  If 'n' is set, exactly
        n servers are chosen.  If 'approx' is set, approximately 'approx'
-       servers are chosen."""
+       servers are chosen.
+    """
     def __init__(self, n=None, approx=None):
         assert not (n and approx)
         assert n is None or approx is None

Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.177
retrieving revision 1.178
diff -u -d -r1.177 -r1.178
--- ClientMain.py	21 Apr 2004 22:15:38 -0000	1.177
+++ ClientMain.py	2 May 2004 18:45:15 -0000	1.178
@@ -958,18 +958,15 @@
             assert _CLIENT_LOCKFILE
             LOG.debug("Configuring server list")
             self.directory = mixminion.ClientDirectory.ClientDirectory(
-                userdir, ClientDiskLock())
-            self.directory.configure(self.config)
+                config=self.config, diskLock=ClientDiskLock())
             self.directory._installAsKeyIDResolver()
 
         if self.wantDownload:
             assert self.wantClientDirectory
-            timeout = int(self.config['DirectoryServers']['DirectoryTimeout'])
             if self.download != 0:
                 clientLock()
                 try:
-                    self.directory.updateDirectory(forceDownload=self.download,
-                                                   timeout=timeout)
+                    self.directory.update(force=self.download)
                 finally:
                     clientUnlock()
 
@@ -1539,7 +1536,7 @@
     timeout = int(config['DirectoryServers']['DirectoryTimeout'])
     clientLock()
     try:
-        directory.updateDirectory(forceDownload=1, timeout=timeout)
+        directory.update(force=1)
     finally:
         clientUnlock()
     print "Directory updated"

Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.137
retrieving revision 1.138
diff -u -d -r1.137 -r1.138
--- Common.py	23 Mar 2004 05:15:16 -0000	1.137
+++ Common.py	2 May 2004 18:45:15 -0000	1.138
@@ -1481,6 +1481,7 @@
         self.filename = filename
         self.count = 0
         self.fd = None
+        self.rlock = threading.Lock() #DOCDOC
 
     def getContents(self):
         """Return the contents of the lock file."""
@@ -1496,22 +1497,26 @@
            defaults to blocking=1.  This should change. XXXX008
         """
 
-        if self.count > 0:
-            self.count += 1
-            return
-
-        assert self.fd is None
-        self.fd = os.open(self.filename, os.O_RDWR|os.O_CREAT, mode)
+        self.rlock.acquire()
         try:
-            self._lock(self.fd, blocking)
-            self.count += 1
-            os.write(self.fd, contents)
-            if hasattr(os, "fsync"):
-                os.fsync(self.fd)
-        except:
-            os.close(self.fd)
-            self.fd = None
-            raise
+            if self.count > 0:
+                self.count += 1
+                return
+
+            assert self.fd is None
+            self.fd = os.open(self.filename, os.O_RDWR|os.O_CREAT, mode)
+            try:
+                self._lock(self.fd, blocking)
+                self.count += 1
+                os.write(self.fd, contents)
+                if hasattr(os, "fsync"):
+                    os.fsync(self.fd)
+            except:
+                os.close(self.fd)
+                self.fd = None
+                raise
+        finally:
+            self.rlock.release()
 
     def replaceContents(self, contents):
         """Replace the current contents of the lockfile with 'contents',
@@ -1529,24 +1534,28 @@
 
     def release(self):
         """Release the lock."""
-        assert self.fd is not None
-        self.count -= 1
-        if self.count > 0:
-            return
-        try:
-            os.unlink(self.filename)
-        except OSError:
-            pass
-        try:
-            self._unlock(self.fd)
-        except OSError:
-            pass
+        self.rlock.acquire()
         try:
-            os.close(self.fd)
-        except OSError:
-            pass
+            assert self.fd is not None
+            self.count -= 1
+            if self.count > 0:
+                return
+            try:
+                os.unlink(self.filename)
+            except OSError:
+                pass
+            try:
+                self._unlock(self.fd)
+            except OSError:
+                pass
+            try:
+                os.close(self.fd)
+            except OSError:
+                pass
 
-        self.fd = None
+            self.fd = None
+        finally:
+            self.rlock.release()
 
     def _lock(self, fd, blocking):
         """Compatibility wrapper to implement file locking for posix and win32

Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.85
retrieving revision 1.86
diff -u -d -r1.85 -r1.86
--- Config.py	26 Apr 2004 16:55:45 -0000	1.85
+++ Config.py	2 May 2004 18:45:15 -0000	1.86
@@ -1056,6 +1056,14 @@
         """DOCDOC"""
         return 0
 
+    def getUserDirectory(self):
+        """DOCDOC"""
+        return os.path.expanduser(self["User"].get("UserDir",DEFAULT_USER_DIR))
+
+    def getDirectoryRoot(self):
+        """DOCDOC"""
+        return self.getUserDirectory()
+
 def _validateHostSection(sec):
     """Helper function: Makes sure that the shared [Host] section is correct;
        raise ConfigError if it isn't"""

Index: ThreadUtils.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ThreadUtils.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- ThreadUtils.py	6 Mar 2004 00:04:38 -0000	1.2
+++ ThreadUtils.py	2 May 2004 18:45:15 -0000	1.3
@@ -92,6 +92,16 @@
             raise AssertionError # unreached, appease pychecker
 
 #----------------------------------------------------------------------
+class DummyLock:
+    """DOCDOC"""
+    def __init__(self):
+        pass
+    def acquire(self):
+        pass
+    def release(self):
+        pass
+
+#----------------------------------------------------------------------
 # RW locks
 #
 # Adapted from the msrw class in the sync.py module in the Python
@@ -117,7 +127,8 @@
         self.rwOK = threading.Lock()
         self.nr = 0  # number readers actively reading (not just waiting)
         self.nw = 0  # number writers either waiting to write or writing
-        self.writing = 0  # 1 iff some thread is writing
+        self.writer = 0  # thread_ident iff some thread is writing
+        self.write_depth = 0 # for recursive write_in invocation.
         # map from each current reader's thread_ident to recursion depth.
         self.readers = {}
 
@@ -131,14 +142,15 @@
         self.rwOK.acquire()
         try:
             ident = _get_ident()
+            if self.writer == ident:
+                raise ValueError("Tried to acquire read lock while holding write lock.")
+
             try:
                 self.readers[ident] += 1
                 self.nr += 1
                 return
             except KeyError:
                 pass
-            #if self.readers.has_key(ident):
-            #    raise ValueError("RWLock.read_in called recursively.")
 
             while self.nw:
                 self.readOK.wait()
@@ -152,9 +164,10 @@
            activate the writers (if any)."""
         self.rwOK.acquire()
         try:
+            ident = _get_ident()
+
             if self.nr <= 0:
                 raise ValueError, '.read_out() invoked without an active reader'
-            ident = _get_ident()
             try:
                 n = self.readers[ident]
             except KeyError:
@@ -165,10 +178,6 @@
                 self.readers[ident] = n-1
                 self.nr -= 1
                 return
-            #try:
-            #    del self.readers[_get_ident()]
-            #except KeyError:
-            #    raise ValueError("read_out called without matching read_in.")
 
             self.nr = self.nr - 1
             if self.nr == 0:
@@ -182,12 +191,17 @@
         """
         self.rwOK.acquire()
         try:
-            if self.readers.has_key(_get_ident()):
+            ident = _get_ident()
+            if self.readers.has_key(ident):
                 raise ValueError("write_in called while acting as reader")
+            if self.writer == ident:
+                self.write_depth += 1
+                return
             self.nw = self.nw + 1
-            while self.writing or self.nr:
+            while self.writer or self.nr:
                 self.writeOK.wait()
-            self.writing = 1
+            self.writer = ident
+            self.write_depth = 1
         finally:
             self.rwOK.release()
 
@@ -195,10 +209,16 @@
         """Release the lock for writing."""
         self.rwOK.acquire()
         try:
-            if not self.writing:
+            ident = _get_ident()
+            if not self.writer:
                 raise ValueError, \
                       '.write_out() invoked without an active writer'
-            self.writing = 0
+            if self.writer != ident:
+                raise ValueError("write_out() called by non-writer")
+            self.write_depth -= 1
+            if self.write_depth > 0:
+                return
+            self.writer = 0
             self.nw = self.nw - 1
             if self.nw:
                 self.writeOK.notify()
@@ -211,9 +231,13 @@
         """Simultaneously release the lock as a writer, and become a reader."""
         self.rwOK.acquire()
         try:
-            if not self.writing:
+            if not self.writer:
                 raise ValueError, \
                       '.write_to_read() invoked without an active writer'
+            if self.writer != ident:
+                raise ValueError("write_out() called by non-writer")
+            assert self.write_depth == 1
+            self.write_depth -= 1
             self.writing = 0
             self.nw = self.nw - 1
             self.nr = self.nr + 1

Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.194
retrieving revision 1.195
diff -u -d -r1.194 -r1.195
--- test.py	19 Apr 2004 03:47:16 -0000	1.194
+++ test.py	2 May 2004 18:45:16 -0000	1.195
@@ -4272,7 +4272,7 @@
             self.assert_(os.path.exists(c[0]) and c[0].endswith("/rm"))
             self.assertEquals(c[1], [])
             self.assertEquals(C._parseCommand("/bin/ls"), ("/bin/ls", []))
-            self.failUnless(C._parseCommand("python")[0] is not None)
+            self.failUnless(C._parseCommand(sys.executable)[0] is not None)
 
         # Base64
         self.assertEquals(C._parseBase64(" YW\nJj"), "abc")
@@ -6570,9 +6570,11 @@
         eq = self.assertEquals
         neq = self.assertNotEquals
         ServerInfo = mixminion.ServerInfo.ServerInfo
-
         dirname = mix_mktemp()
-        ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+        config = mixminion.Config.ClientConfig(
+            string="[User]\nUserDir: %s\n"%dirname)
+
+        ks = mixminion.ClientDirectory.ClientDirectory(config)
 
         ## Write the descriptors to disk.
         impdirname = self.writeDescriptorsToDisk()
@@ -6602,7 +6604,7 @@
             self.assertRaises(MixError, ks.getServerInfo, "Joe", startAt=now,
                               endAt=now+6*oneDay, strict=1)
             if i in (0,1,2):
-                ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+                ks = mixminion.ClientDirectory.ClientDirectory(config)
             if i == 1:
                 ks.rescan()
             if i == 2:
@@ -6642,7 +6644,7 @@
         mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint
 
         # Reload the directory.
-        ks.updateDirectory(now=now)
+        ks.update(now=now,force=1)
 
         for i in 0,1,2,3:
             self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
@@ -6651,20 +6653,20 @@
                               edesc["Bob"][4])
 
             if i in (0,1,2):
-                ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+                ks = mixminion.ClientDirectory.ClientDirectory(config)
             if i == 1:
                 ks.rescan()
             if i == 2:
                 ks.rescan(force=1)
 
-        replaceFunction(ks, 'downloadDirectory')
+        replaceFunction(ks.store.bases[0], 'downloadDirectory')
 
         # Now make sure that update is properly zealous.
-        ks.updateDirectory(now=now)
+        ks.update(now=now)
         self.assertEquals([], getReplacedFunctionCallLog())
-        ks.updateDirectory(now=now, forceDownload=1)
+        ks.update(now=now, force=1)
         self.assertEquals(1, len(getReplacedFunctionCallLog()))
-        ks.updateDirectory(now=now+oneDay+60)
+        ks.update(now=now+oneDay+60)
         self.assertEquals(2, len(getReplacedFunctionCallLog()))
         undoReplacedAttributes()
         clearReplacedFunctionCallLog()
@@ -6676,7 +6678,7 @@
              ("Fred1", "Fred2", "Lola2", "Alice0", "Alice1",
               "Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
         mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
-        ks.updateDirectory(forceDownload=1)
+        ks.update(force=1)
         # Previous entries.
         self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
         self.assertSameSD(ks.getServerInfo("Bob"), edesc["Bob"][3])
@@ -6742,9 +6744,11 @@
             neq(p[1].getNickname(), "Joe")
 
             # 2a.1. (Blocking some servers)
-            ks.configure({"Security" : { "BlockServers" : [["Joe"]],
-                                         "BlockEntries" : [["Alice"]],
-                                         "BlockExits" : [["Bob"]] } })
+            configBlock = mixminion.Config.ClientConfig(string=(
+                "[User]\nUserDir: %s\n[Security]\nBlockServers: Joe\n"
+                "BlockEntries: Alice\nBlockExits: Bob\n" % dirname))
+            ks.configure(configBlock)
+
             for _ in xrange(100):
                 p = ks.getPath([None]*4)
                 eq(4, len(p))
@@ -6753,9 +6757,13 @@
                 self.assertNotEquals("Bob", p[3])
                 self.assertNotIn("Joe", p)
 
-            ks.configure({})
+            ks.configure(config)
             # 2b. With 3 <= servers < length
-            ks2 = mixminion.ClientDirectory.ClientDirectory(mix_mktemp())
+            dirname2 = mix_mktemp()
+            config2 = mixminion.Config.ClientConfig(
+                string="[User]\nUserDir: %s\n"%dirname2)
+
+            ks2 = mixminion.ClientDirectory.ClientDirectory(config2)
             ks2.importFromFile(os.path.join(impdirname, "Joe0"))
             ks2.importFromFile(os.path.join(impdirname, "Alice0"))
             ks2.importFromFile(os.path.join(impdirname, "Lisa1"))
@@ -6970,7 +6978,7 @@
 
         ## Now try clean()
         ks.clean() # Should do nothing.
-        ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+        ks = mixminion.ClientDirectory.ClientDirectory(config)
         ks.clean(now=now+oneDay*500) # Should zap all of imported servers.
         raises(MixError, ks.getServerInfo, "Lola", strict=1)
         ks.getServerInfo
@@ -6979,7 +6987,10 @@
         from mixminion.ClientDirectory import compressFeatureMap
         from mixminion.ClientDirectory import formatFeatureMap
         d = self.writeDescriptorsToDisk()
-        direc = mixminion.ClientDirectory.ClientDirectory(mix_mktemp())
+        dirname = mix_mktemp()
+        config = mixminion.Config.ClientConfig(
+            string="[User]\nUserDir: %s\n"%dirname)
+        direc = mixminion.ClientDirectory.ClientDirectory(config)
         self.loadDirectory(direc, d)
         edesc = getExampleServerDescriptors()
 
@@ -7075,7 +7086,7 @@
               "Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
         mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
         mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint
-        direc.updateDirectory(now=now)
+        direc.update(now=now)
 
     def assertSameSD(self, s1, s2):
         self.assert_(self.isSameServerDesc(s1,s2))
@@ -7184,8 +7195,7 @@
         client = mixminion.ClientMain.MixminionClient(usercfg)
 
         # Create a directory...
-        dirname = mix_mktemp()
-        directory = mixminion.ClientDirectory.ClientDirectory(dirname)
+        directory = mixminion.ClientDirectory.ClientDirectory(usercfg)
 
         edesc = getExampleServerDescriptors()
         fname = mix_mktemp()
@@ -7515,7 +7525,7 @@
     tc = loader.loadTestsFromTestCase
 
     if 0:
-        suite.addTest(tc(MMTPTests))
+        suite.addTest(tc(ClientMainTests))
         return suite
     testClasses = [MiscTests,
                    MinionlibCryptoTests,