[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[minion-cvs] Debug refactored ClientDirectory. Now it will be easie...
Update of /home/minion/cvsroot/src/minion/lib/mixminion
In directory moria.mit.edu:/tmp/cvs-serv8278/lib/mixminion
Modified Files:
ClientDirectory.py ClientMain.py Common.py Config.py
ThreadUtils.py test.py
Log Message:
Debug refactored ClientDirectory. Now it will be easier to use it for pinging.
Index: ClientDirectory.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientDirectory.py,v
retrieving revision 1.38
retrieving revision 1.39
diff -u -d -r1.38 -r1.39
--- ClientDirectory.py 26 Apr 2004 16:55:45 -0000 1.38
+++ ClientDirectory.py 2 May 2004 18:45:15 -0000 1.39
@@ -9,7 +9,7 @@
"""
__all__ = [ 'ClientDirectory', 'parsePath', 'parseAddress',
- 'GotInvalidDirectoryError' ]
+ 'DirectoryDownloadError', 'GotInvalidDirectoryError' ]
import cPickle
import errno
@@ -35,17 +35,17 @@
from mixminion.Packet import MBOX_TYPE, SMTP_TYPE, DROP_TYPE, FRAGMENT_TYPE, \
parseMBOXInfo, parseRelayInfoByType, parseSMTPInfo, ParseError, \
[...2342 lines suppressed...]
@@ -2793,7 +1712,8 @@
return self.getMinLength()
class ServerPathElement(PathElement):
- """A path element for a single server specified by filename or nickname"""
+ """A path element for a single server specified by filename or nickname
+ """
def __init__(self, nickname):
self.nickname = nickname
def validate(self, directory, start, end):
@@ -2832,7 +1752,8 @@
class RandomServersPathElement(PathElement):
"""A path element for randomly chosen servers. If 'n' is set, exactly
n servers are chosen. If 'approx' is set, approximately 'approx'
- servers are chosen."""
+ servers are chosen.
+ """
def __init__(self, n=None, approx=None):
assert not (n and approx)
assert n is None or approx is None
Index: ClientMain.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ClientMain.py,v
retrieving revision 1.177
retrieving revision 1.178
diff -u -d -r1.177 -r1.178
--- ClientMain.py 21 Apr 2004 22:15:38 -0000 1.177
+++ ClientMain.py 2 May 2004 18:45:15 -0000 1.178
@@ -958,18 +958,15 @@
assert _CLIENT_LOCKFILE
LOG.debug("Configuring server list")
self.directory = mixminion.ClientDirectory.ClientDirectory(
- userdir, ClientDiskLock())
- self.directory.configure(self.config)
+ config=self.config, diskLock=ClientDiskLock())
self.directory._installAsKeyIDResolver()
if self.wantDownload:
assert self.wantClientDirectory
- timeout = int(self.config['DirectoryServers']['DirectoryTimeout'])
if self.download != 0:
clientLock()
try:
- self.directory.updateDirectory(forceDownload=self.download,
- timeout=timeout)
+ self.directory.update(force=self.download)
finally:
clientUnlock()
@@ -1539,7 +1536,7 @@
timeout = int(config['DirectoryServers']['DirectoryTimeout'])
clientLock()
try:
- directory.updateDirectory(forceDownload=1, timeout=timeout)
+ directory.update(force=1)
finally:
clientUnlock()
print "Directory updated"
Index: Common.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Common.py,v
retrieving revision 1.137
retrieving revision 1.138
diff -u -d -r1.137 -r1.138
--- Common.py 23 Mar 2004 05:15:16 -0000 1.137
+++ Common.py 2 May 2004 18:45:15 -0000 1.138
@@ -1481,6 +1481,7 @@
self.filename = filename
self.count = 0
self.fd = None
+ self.rlock = threading.Lock() #DOCDOC
def getContents(self):
"""Return the contents of the lock file."""
@@ -1496,22 +1497,26 @@
defaults to blocking=1. This should change. XXXX008
"""
- if self.count > 0:
- self.count += 1
- return
-
- assert self.fd is None
- self.fd = os.open(self.filename, os.O_RDWR|os.O_CREAT, mode)
+ self.rlock.acquire()
try:
- self._lock(self.fd, blocking)
- self.count += 1
- os.write(self.fd, contents)
- if hasattr(os, "fsync"):
- os.fsync(self.fd)
- except:
- os.close(self.fd)
- self.fd = None
- raise
+ if self.count > 0:
+ self.count += 1
+ return
+
+ assert self.fd is None
+ self.fd = os.open(self.filename, os.O_RDWR|os.O_CREAT, mode)
+ try:
+ self._lock(self.fd, blocking)
+ self.count += 1
+ os.write(self.fd, contents)
+ if hasattr(os, "fsync"):
+ os.fsync(self.fd)
+ except:
+ os.close(self.fd)
+ self.fd = None
+ raise
+ finally:
+ self.rlock.release()
def replaceContents(self, contents):
"""Replace the current contents of the lockfile with 'contents',
@@ -1529,24 +1534,28 @@
def release(self):
"""Release the lock."""
- assert self.fd is not None
- self.count -= 1
- if self.count > 0:
- return
- try:
- os.unlink(self.filename)
- except OSError:
- pass
- try:
- self._unlock(self.fd)
- except OSError:
- pass
+ self.rlock.acquire()
try:
- os.close(self.fd)
- except OSError:
- pass
+ assert self.fd is not None
+ self.count -= 1
+ if self.count > 0:
+ return
+ try:
+ os.unlink(self.filename)
+ except OSError:
+ pass
+ try:
+ self._unlock(self.fd)
+ except OSError:
+ pass
+ try:
+ os.close(self.fd)
+ except OSError:
+ pass
- self.fd = None
+ self.fd = None
+ finally:
+ self.rlock.release()
def _lock(self, fd, blocking):
"""Compatibility wrapper to implement file locking for posix and win32
Index: Config.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/Config.py,v
retrieving revision 1.85
retrieving revision 1.86
diff -u -d -r1.85 -r1.86
--- Config.py 26 Apr 2004 16:55:45 -0000 1.85
+++ Config.py 2 May 2004 18:45:15 -0000 1.86
@@ -1056,6 +1056,14 @@
"""DOCDOC"""
return 0
+ def getUserDirectory(self):
+ """DOCDOC"""
+ return os.path.expanduser(self["User"].get("UserDir",DEFAULT_USER_DIR))
+
+ def getDirectoryRoot(self):
+ """DOCDOC"""
+ return self.getUserDirectory()
+
def _validateHostSection(sec):
"""Helper function: Makes sure that the shared [Host] section is correct;
raise ConfigError if it isn't"""
Index: ThreadUtils.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/ThreadUtils.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- ThreadUtils.py 6 Mar 2004 00:04:38 -0000 1.2
+++ ThreadUtils.py 2 May 2004 18:45:15 -0000 1.3
@@ -92,6 +92,16 @@
raise AssertionError # unreached, appease pychecker
#----------------------------------------------------------------------
+class DummyLock:
+ """DOCDOC"""
+ def __init__(self):
+ pass
+ def acquire(self):
+ pass
+ def release(self):
+ pass
+
+#----------------------------------------------------------------------
# RW locks
#
# Adapted from the msrw class in the sync.py module in the Python
@@ -117,7 +127,8 @@
self.rwOK = threading.Lock()
self.nr = 0 # number readers actively reading (not just waiting)
self.nw = 0 # number writers either waiting to write or writing
- self.writing = 0 # 1 iff some thread is writing
+ self.writer = 0 # thread_ident iff some thread is writing
+ self.write_depth = 0 # for recursive write_in invocation.
# map from each current reader's thread_ident to recursion depth.
self.readers = {}
@@ -131,14 +142,15 @@
self.rwOK.acquire()
try:
ident = _get_ident()
+ if self.writer == ident:
+ raise ValueError("Tried to acquire read lock while holding write lock.")
+
try:
self.readers[ident] += 1
self.nr += 1
return
except KeyError:
pass
- #if self.readers.has_key(ident):
- # raise ValueError("RWLock.read_in called recursively.")
while self.nw:
self.readOK.wait()
@@ -152,9 +164,10 @@
activate the writers (if any)."""
self.rwOK.acquire()
try:
+ ident = _get_ident()
+
if self.nr <= 0:
raise ValueError, '.read_out() invoked without an active reader'
- ident = _get_ident()
try:
n = self.readers[ident]
except KeyError:
@@ -165,10 +178,6 @@
self.readers[ident] = n-1
self.nr -= 1
return
- #try:
- # del self.readers[_get_ident()]
- #except KeyError:
- # raise ValueError("read_out called without matching read_in.")
self.nr = self.nr - 1
if self.nr == 0:
@@ -182,12 +191,17 @@
"""
self.rwOK.acquire()
try:
- if self.readers.has_key(_get_ident()):
+ ident = _get_ident()
+ if self.readers.has_key(ident):
raise ValueError("write_in called while acting as reader")
+ if self.writer == ident:
+ self.write_depth += 1
+ return
self.nw = self.nw + 1
- while self.writing or self.nr:
+ while self.writer or self.nr:
self.writeOK.wait()
- self.writing = 1
+ self.writer = ident
+ self.write_depth = 1
finally:
self.rwOK.release()
@@ -195,10 +209,16 @@
"""Release the lock for writing."""
self.rwOK.acquire()
try:
- if not self.writing:
+ ident = _get_ident()
+ if not self.writer:
raise ValueError, \
'.write_out() invoked without an active writer'
- self.writing = 0
+ if self.writer != ident:
+ raise ValueError("write_out() called by non-writer")
+ self.write_depth -= 1
+ if self.write_depth > 0:
+ return
+ self.writer = 0
self.nw = self.nw - 1
if self.nw:
self.writeOK.notify()
@@ -211,9 +231,13 @@
"""Simultaneously release the lock as a writer, and become a reader."""
self.rwOK.acquire()
try:
- if not self.writing:
+ if not self.writer:
raise ValueError, \
'.write_to_read() invoked without an active writer'
+ if self.writer != ident:
+ raise ValueError("write_out() called by non-writer")
+ assert self.write_depth == 1
+ self.write_depth -= 1
self.writing = 0
self.nw = self.nw - 1
self.nr = self.nr + 1
Index: test.py
===================================================================
RCS file: /home/minion/cvsroot/src/minion/lib/mixminion/test.py,v
retrieving revision 1.194
retrieving revision 1.195
diff -u -d -r1.194 -r1.195
--- test.py 19 Apr 2004 03:47:16 -0000 1.194
+++ test.py 2 May 2004 18:45:16 -0000 1.195
@@ -4272,7 +4272,7 @@
self.assert_(os.path.exists(c[0]) and c[0].endswith("/rm"))
self.assertEquals(c[1], [])
self.assertEquals(C._parseCommand("/bin/ls"), ("/bin/ls", []))
- self.failUnless(C._parseCommand("python")[0] is not None)
+ self.failUnless(C._parseCommand(sys.executable)[0] is not None)
# Base64
self.assertEquals(C._parseBase64(" YW\nJj"), "abc")
@@ -6570,9 +6570,11 @@
eq = self.assertEquals
neq = self.assertNotEquals
ServerInfo = mixminion.ServerInfo.ServerInfo
-
dirname = mix_mktemp()
- ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+ config = mixminion.Config.ClientConfig(
+ string="[User]\nUserDir: %s\n"%dirname)
+
+ ks = mixminion.ClientDirectory.ClientDirectory(config)
## Write the descriptors to disk.
impdirname = self.writeDescriptorsToDisk()
@@ -6602,7 +6604,7 @@
self.assertRaises(MixError, ks.getServerInfo, "Joe", startAt=now,
endAt=now+6*oneDay, strict=1)
if i in (0,1,2):
- ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+ ks = mixminion.ClientDirectory.ClientDirectory(config)
if i == 1:
ks.rescan()
if i == 2:
@@ -6642,7 +6644,7 @@
mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint
# Reload the directory.
- ks.updateDirectory(now=now)
+ ks.update(now=now,force=1)
for i in 0,1,2,3:
self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
@@ -6651,20 +6653,20 @@
edesc["Bob"][4])
if i in (0,1,2):
- ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+ ks = mixminion.ClientDirectory.ClientDirectory(config)
if i == 1:
ks.rescan()
if i == 2:
ks.rescan(force=1)
- replaceFunction(ks, 'downloadDirectory')
+ replaceFunction(ks.store.bases[0], 'downloadDirectory')
# Now make sure that update is properly zealous.
- ks.updateDirectory(now=now)
+ ks.update(now=now)
self.assertEquals([], getReplacedFunctionCallLog())
- ks.updateDirectory(now=now, forceDownload=1)
+ ks.update(now=now, force=1)
self.assertEquals(1, len(getReplacedFunctionCallLog()))
- ks.updateDirectory(now=now+oneDay+60)
+ ks.update(now=now+oneDay+60)
self.assertEquals(2, len(getReplacedFunctionCallLog()))
undoReplacedAttributes()
clearReplacedFunctionCallLog()
@@ -6676,7 +6678,7 @@
("Fred1", "Fred2", "Lola2", "Alice0", "Alice1",
"Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
- ks.updateDirectory(forceDownload=1)
+ ks.update(force=1)
# Previous entries.
self.assertSameSD(ks.getServerInfo("Alice"), edesc["Alice"][0])
self.assertSameSD(ks.getServerInfo("Bob"), edesc["Bob"][3])
@@ -6742,9 +6744,11 @@
neq(p[1].getNickname(), "Joe")
# 2a.1. (Blocking some servers)
- ks.configure({"Security" : { "BlockServers" : [["Joe"]],
- "BlockEntries" : [["Alice"]],
- "BlockExits" : [["Bob"]] } })
+ configBlock = mixminion.Config.ClientConfig(string=(
+ "[User]\nUserDir: %s\n[Security]\nBlockServers: Joe\n"
+ "BlockEntries: Alice\nBlockExits: Bob\n" % dirname))
+ ks.configure(configBlock)
+
for _ in xrange(100):
p = ks.getPath([None]*4)
eq(4, len(p))
@@ -6753,9 +6757,13 @@
self.assertNotEquals("Bob", p[3])
self.assertNotIn("Joe", p)
- ks.configure({})
+ ks.configure(config)
# 2b. With 3 <= servers < length
- ks2 = mixminion.ClientDirectory.ClientDirectory(mix_mktemp())
+ dirname2 = mix_mktemp()
+ config2 = mixminion.Config.ClientConfig(
+ string="[User]\nUserDir: %s\n"%dirname2)
+
+ ks2 = mixminion.ClientDirectory.ClientDirectory(config2)
ks2.importFromFile(os.path.join(impdirname, "Joe0"))
ks2.importFromFile(os.path.join(impdirname, "Alice0"))
ks2.importFromFile(os.path.join(impdirname, "Lisa1"))
@@ -6970,7 +6978,7 @@
## Now try clean()
ks.clean() # Should do nothing.
- ks = mixminion.ClientDirectory.ClientDirectory(dirname)
+ ks = mixminion.ClientDirectory.ClientDirectory(config)
ks.clean(now=now+oneDay*500) # Should zap all of imported servers.
raises(MixError, ks.getServerInfo, "Lola", strict=1)
ks.getServerInfo
@@ -6979,7 +6987,10 @@
from mixminion.ClientDirectory import compressFeatureMap
from mixminion.ClientDirectory import formatFeatureMap
d = self.writeDescriptorsToDisk()
- direc = mixminion.ClientDirectory.ClientDirectory(mix_mktemp())
+ dirname = mix_mktemp()
+ config = mixminion.Config.ClientConfig(
+ string="[User]\nUserDir: %s\n"%dirname)
+ direc = mixminion.ClientDirectory.ClientDirectory(config)
self.loadDirectory(direc, d)
edesc = getExampleServerDescriptors()
@@ -7075,7 +7086,7 @@
"Bob3", "Bob4", "Lisa1", "Lisa2") ], identity)
mixminion.ClientDirectory.MIXMINION_DIRECTORY_URL = fileURL(fname)
mixminion.ClientDirectory.MIXMINION_DIRECTORY_FINGERPRINT = fingerprint
- direc.updateDirectory(now=now)
+ direc.update(now=now)
def assertSameSD(self, s1, s2):
self.assert_(self.isSameServerDesc(s1,s2))
@@ -7184,8 +7195,7 @@
client = mixminion.ClientMain.MixminionClient(usercfg)
# Create a directory...
- dirname = mix_mktemp()
- directory = mixminion.ClientDirectory.ClientDirectory(dirname)
+ directory = mixminion.ClientDirectory.ClientDirectory(usercfg)
edesc = getExampleServerDescriptors()
fname = mix_mktemp()
@@ -7515,7 +7525,7 @@
tc = loader.loadTestsFromTestCase
if 0:
- suite.addTest(tc(MMTPTests))
+ suite.addTest(tc(ClientMainTests))
return suite
testClasses = [MiscTests,
MinionlibCryptoTests,