[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r22295: {arm} Util for improved system call handling. added: system tools (in arm/trunk: interface util)
Author: atagar
Date: 2010-05-07 16:25:49 +0000 (Fri, 07 May 2010)
New Revision: 22295
Added:
arm/trunk/util/sysTools.py
Modified:
arm/trunk/interface/controller.py
arm/trunk/interface/cpuMemMonitor.py
arm/trunk/interface/fileDescriptorPopup.py
arm/trunk/interface/headerPanel.py
arm/trunk/interface/logPanel.py
arm/trunk/util/__init__.py
arm/trunk/util/conf.py
arm/trunk/util/connections.py
arm/trunk/util/hostnames.py
arm/trunk/util/panel.py
arm/trunk/util/uiTools.py
Log:
Util for improved system call handling.
added: system tools util providing: simplified usage, suppression of leaks to stdout, logging, and optional caching
change: caching ps calls for several seconds (lowers the call volume but also reduces the fidelity of the resource graph)
fix: preventing 'command unavailable' error messages from going to stdout, which disrupts the display (caught by sid77)
fix: bug in defaulting the connection resolver to something predetermined to be available
fix: connection resolver was being initiated while in blind mode
Modified: arm/trunk/interface/controller.py
===================================================================
--- arm/trunk/interface/controller.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/interface/controller.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -7,7 +7,6 @@
"""
import re
-import os
import math
import time
import curses
@@ -23,7 +22,7 @@
import descriptorPopup
import fileDescriptorPopup
-from util import log, connections, hostnames, panel, uiTools
+from util import log, connections, hostnames, panel, sysTools, uiTools
import bandwidthMonitor
import cpuMemMonitor
import connCountMonitor
@@ -324,52 +323,45 @@
# gets pid of tor instance with control port open
torPid = None # None if couldn't be resolved (provides error later)
- pidOfCall = os.popen("pidof tor 2> /dev/null")
try:
# gets pid if there's only one possability
- results = pidOfCall.readlines()
+ results = sysTools.call("pidof tor")
if len(results) == 1 and len(results[0].split()) == 1: torPid = results[0].strip()
except IOError: pass # pid call failed
- pidOfCall.close()
if not torPid:
try:
# uses netstat to identify process with open control port (might not
# work if tor's being run as a different user due to permissions)
- netstatCall = os.popen("netstat -npl 2> /dev/null | grep 127.0.0.1:%s 2> /dev/null" % conn.get_option("ControlPort")[0][1])
- results = netstatCall.readlines()
+ results = sysTools.call("netstat -npl | grep 127.0.0.1:%s" % conn.get_option("ControlPort")[0][1])
if len(results) == 1:
results = results[0].split()[6] # process field (ex. "7184/tor")
torPid = results[:results.find("/")]
except (IOError, socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed): pass # netstat or control port calls failed
- netstatCall.close()
if not torPid:
try:
# third try, use ps if there's only one possability
- psCall = os.popen("ps -o pid -C tor 2> /dev/null")
- results = psCall.readlines()
+ results = sysTools.call("ps -o pid -C tor")
if len(results) == 2 and len(results[0].split()) == 1: torPid = results[1].strip()
except IOError: pass # ps call failed
- psCall.close()
try:
confLocation = conn.get_info("config-file")["config-file"]
if confLocation[0] != "/":
# relative path - attempt to add process pwd
try:
- pwdxCall = os.popen("pwdx %s 2> /dev/null" % torPid)
- results = pwdxCall.readlines()
+ results = sysTools.call("pwdx %s" % torPid)
if len(results) == 1 and len(results[0].split()) == 2: confLocation = "%s/%s" % (results[0].split()[1], confLocation)
except IOError: pass # pwdx call failed
- pwdxCall.close()
except (socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed):
confLocation = ""
# minor refinements for connection resolver
- resolver = connections.getResolver("tor")
- if torPid: resolver.processPid = torPid # helps narrow connection results
+ if not isBlindMode:
+ resolver = connections.getResolver("tor")
+ if torPid: resolver.processPid = torPid # helps narrow connection results
# hack to display a better (arm specific) notice if all resolvers fail
connections.RESOLVER_FINAL_FAILURE_MSG += " (connection related portions of the monitor won't function)"
Modified: arm/trunk/interface/cpuMemMonitor.py
===================================================================
--- arm/trunk/interface/cpuMemMonitor.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/interface/cpuMemMonitor.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -2,11 +2,10 @@
# cpuMemMonitor.py -- Tracks cpu and memory usage of Tor.
# Released under the GPL v3 (http://www.gnu.org/licenses/gpl.html)
-import os
import time
from TorCtl import TorCtl
-from util import uiTools
+from util import sysTools, uiTools
import graphPanel
class CpuMemMonitor(graphPanel.GraphStats, TorCtl.PostEventListener):
@@ -24,25 +23,29 @@
def bandwidth_event(self, event):
# doesn't use events but this keeps it in sync with the bandwidth panel
# (and so it stops if Tor stops
- if self.headerPanel.lastUpdate + 1 >= time.time():
+ # TODO: ok, screw it - the number of ps calls this makes is ridicuous
+ # compared to how frequently it changes - now caching for five seconds
+ # (note this during the rewrite that its fidelity isn't at the second
+ # level)
+ if self.headerPanel.lastUpdate + 5 >= time.time():
# reuses ps results if recent enough
self._processEvent(float(self.headerPanel.vals["%cpu"]), float(self.headerPanel.vals["rss"]) / 1024.0)
else:
# cached results stale - requery ps
- psCall = os.popen('ps -p %s -o %s 2> /dev/null' % (self.headerPanel.vals["pid"], "%cpu,rss"))
- try:
- sampling = psCall.read().strip().split()[2:]
- psCall.close()
-
- if len(sampling) < 2:
- # either ps failed or returned no tor instance, register error
- raise IOError()
- else:
- self._processEvent(float(sampling[0]), float(sampling[1]) / 1024.0)
- except IOError:
- # ps call failed - we need to register something (otherwise timescale
- # would be thrown off) so keep old results
+ sampling = []
+ psCall = None
+ if self.headerPanel.vals["pid"]:
+ psCall = sysTools.call("ps -p %s -o %s" % (self.headerPanel.vals["pid"], "%cpu,rss"), 5, True)
+ if psCall and len(psCall) >= 2: sampling = psCall[1].strip().split()
+
+ if len(sampling) < 2:
+ # either ps failed or returned no tor instance, register error
+ # ps call failed (returned no tor instance or registered an error) -
+ # we need to register something (otherwise timescale would be thrown
+ # off) so keep old results
self._processEvent(self.lastPrimary, self.lastSecondary)
+ else:
+ self._processEvent(float(sampling[0]), float(sampling[1]) / 1024.0)
def getTitle(self, width):
return "System Resources:"
Modified: arm/trunk/interface/fileDescriptorPopup.py
===================================================================
--- arm/trunk/interface/fileDescriptorPopup.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/interface/fileDescriptorPopup.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -5,7 +5,7 @@
import os
import curses
-from util import panel, uiTools
+from util import panel, sysTools, uiTools
class PopupProperties:
"""
@@ -27,13 +27,10 @@
# retrieves list of open files, options are:
# n = no dns lookups, p = by pid, -F = show fields (L = login name, n = opened files)
- lsofCall = os.popen3("lsof -np %s -F Ln 2> /dev/null" % torPid)
- results = lsofCall[1].readlines()
- errResults = lsofCall[2].readlines()
+ # TODO: better rewrite to take advantage of sysTools
- # checks if lsof was unavailable
- if "not found" in "".join(errResults):
- raise Exception("error: lsof is unavailable")
+ if not sysTools.isAvailable("lsof"): raise Exception("error: lsof is unavailable")
+ results = sysTools.call("lsof -np %s -F Ln" % torPid)
# if we didn't get any results then tor's probably closed (keep defaults)
if len(results) == 0: return
@@ -77,13 +74,17 @@
results = ulimitCall.readlines()
if len(results) == 0: raise Exception("error: ulimit is unavailable")
self.fdLimit = int(results[0])
+
+ # can't use sysTools for this call because ulimit isn't in the path...
+ # so how the **** am I to detect if it's available!
+ #if not sysTools.isAvailable("ulimit"): raise Exception("error: ulimit is unavailable")
+ #results = sysTools.call("ulimit -Hn")
+ #if len(results) == 0: raise Exception("error: ulimit call failed")
+ #self.fdLimit = int(results[0])
except Exception, exc:
# problem arose in calling or parsing lsof or ulimit calls
self.errorMsg = str(exc)
finally:
- lsofCall[0].close()
- lsofCall[1].close()
- lsofCall[2].close()
if ulimitCall: ulimitCall.close()
def handleKey(self, key, height):
Modified: arm/trunk/interface/headerPanel.py
===================================================================
--- arm/trunk/interface/headerPanel.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/interface/headerPanel.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -7,7 +7,7 @@
import socket
from TorCtl import TorCtl
-from util import panel, uiTools
+from util import panel, sysTools, uiTools
# minimum width for which panel attempts to double up contents (two columns to
# better use screen real estate)
@@ -247,18 +247,18 @@
except (socket.error, TorCtl.ErrorReply, TorCtl.TorCtlClosed): pass
psParams = ["%cpu", "rss", "%mem", "etime"]
+ sampling = []
if self.vals["pid"]:
# ps call provides header followed by params for tor
- psCall = os.popen('ps -p %s -o %s 2> /dev/null' % (self.vals["pid"], ",".join(psParams)))
-
- try: sampling = psCall.read().strip().split()[len(psParams):]
- except IOError: sampling = [] # ps call failed
- psCall.close()
- else:
- sampling = [] # no pid known - blank fields
+ # this caches the results for five seconds and suppress any exceptions
+ # results are expected to look something like:
+ # %CPU RSS %MEM ELAPSED
+ # 0.3 14096 1.3 29:51
+ psCall = sysTools.call("ps -p %s -o %s" % (self.vals["pid"], ",".join(psParams)), 5, True)
+ if psCall and len(psCall) >= 2: sampling = psCall[1].strip().split()
- if len(sampling) < 4:
- # either ps failed or returned no tor instance, blank information except runtime
+ if len(sampling) < len(psParams):
+ # pid is unknown, ps call failed, or returned no tor instance - blank information except runtime
if "etime" in self.vals: sampling = [""] * (len(psParams) - 1) + [self.vals["etime"]]
else: sampling = [""] * len(psParams)
Modified: arm/trunk/interface/logPanel.py
===================================================================
--- arm/trunk/interface/logPanel.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/interface/logPanel.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -2,13 +2,12 @@
# logPanel.py -- Resources related to Tor event monitoring.
# Released under the GPL v3 (http://www.gnu.org/licenses/gpl.html)
-import os
import time
import curses
from curses.ascii import isprint
from TorCtl import TorCtl
-from util import log, panel, uiTools
+from util import log, panel, sysTools, uiTools
PRE_POPULATE_LOG = True # attempts to retrieve events from log file if available
@@ -111,7 +110,6 @@
# attempts to process events from log file
if PRE_POPULATE_LOG:
previousPauseState = self.isPaused
- tailCall = None
try:
logFileLoc = None
@@ -129,11 +127,11 @@
# trims log to last entries to deal with logs when they're in the GB or TB range
# throws IOError if tail fails (falls to the catch-all later)
+ # TODO: now that this is using sysTools figure out if we can do away with the catch-all...
limit = PRE_POPULATE_MIN_LIMIT if ("DEBUG" in self.loggedEvents or "INFO" in self.loggedEvents) else PRE_POPULATE_MAX_LIMIT
- tailCall = os.popen("tail -n %i %s 2> /dev/null" % (limit, logFileLoc))
# truncates to entries for this tor instance
- lines = tailCall.readlines()
+ lines = sysTools.call("tail -n %i %s" % (limit, logFileLoc))
instanceStart = 0
for i in range(len(lines) - 1, -1, -1):
if "opening log file" in lines[i]:
@@ -152,7 +150,6 @@
finally:
self.setPaused(previousPauseState)
self.eventTimeOverwrite = None
- if tailCall: tailCall.close()
def handleKey(self, key):
# scroll movement
Modified: arm/trunk/util/__init__.py
===================================================================
--- arm/trunk/util/__init__.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/__init__.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -4,5 +4,5 @@
and safely working with curses (hiding some of the gory details).
"""
-__all__ = ["connections", "hostnames", "log", "panel", "uiTools"]
+__all__ = ["connections", "hostnames", "log", "panel", "sysTools", "torTools", "uiTools"]
Modified: arm/trunk/util/conf.py
===================================================================
--- arm/trunk/util/conf.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/conf.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -29,7 +29,7 @@
handle - unique identifier used to access this config instance
"""
- if not handle in CONFS.keys(): CONFS[handle] = Config()
+ if not handle in CONFS: CONFS[handle] = Config()
return CONFS[handle]
class Config():
@@ -64,7 +64,7 @@
"""
self.contentsLock.acquire()
- if key in self.contents.keys(): val = self.contents[key]
+ if key in self.contents: val = self.contents[key]
else: val = default
self.contentsLock.release()
Modified: arm/trunk/util/connections.py
===================================================================
--- arm/trunk/util/connections.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/connections.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -11,12 +11,12 @@
program for 'ss', so this is quite likely to fail there.
"""
-import os
import sys
import time
import threading
import log
+import sysTools
# enums for connection resolution utilities
CMD_NETSTAT, CMD_SS, CMD_LSOF = range(1, 4)
@@ -29,18 +29,18 @@
# tcp 0 0 127.0.0.1:9051 127.0.0.1:53308 ESTABLISHED 9912/tor
# *note: bsd uses a different variant ('-t' => '-p tcp', but worse an
# equivilant -p doesn't exist so this can't function)
-RUN_NETSTAT = "netstat -npt 2> /dev/null | grep %s/%s 2> /dev/null"
+RUN_NETSTAT = "netstat -npt | grep %s/%s"
# n = numeric ports, p = include process
# output:
# ESTAB 0 0 127.0.0.1:9051 127.0.0.1:53308 users:(("tor",9912,20))
# *note: under freebsd this command belongs to a spreadsheet program
-RUN_SS = "ss -np 2> /dev/null | grep \"\\\"%s\\\",%s\" 2> /dev/null"
+RUN_SS = "ss -np | grep \"\\\"%s\\\",%s\""
# n = prevent dns lookups, P = show port numbers (not names), i = ip only
# output:
# tor 9912 atagar 20u IPv4 33453 TCP 127.0.0.1:9051->127.0.0.1:53308
-RUN_LSOF = "lsof -nPi 2> /dev/null | grep \"%s\s*%s.*(ESTABLISHED)\" 2> /dev/null"
+RUN_LSOF = "lsof -nPi | grep \"%s\s*%s.*(ESTABLISHED)\""
RESOLVERS = [] # connection resolvers available via the singleton constructor
RESOLVER_MIN_DEFAULT_LOOKUP = 5 # minimum seconds between lookups (unless overwritten)
@@ -49,9 +49,6 @@
RESOLVER_SERIAL_FAILURE_MSG = "Querying connections with %s failed, trying %s"
RESOLVER_FINAL_FAILURE_MSG = "All connection resolvers failed"
-# mapping of commands to if they're available or not
-CMD_AVAILABLE_CACHE = {}
-
def getConnections(resolutionCmd, processName, processPid = ""):
"""
Retrieves a list of the current connections for a given process, providing a
@@ -69,19 +66,14 @@
processPid - process ID (this helps improve accuracy)
"""
- # first check if the command's available to avoid sending error to stdout
- if not _isAvailable(CMD_STR[resolutionCmd]):
- raise IOError("'%s' is unavailable" % CMD_STR[resolutionCmd])
-
if resolutionCmd == CMD_NETSTAT: cmd = RUN_NETSTAT % (processPid, processName)
elif resolutionCmd == CMD_SS: cmd = RUN_SS % (processName, processPid)
else: cmd = RUN_LSOF % (processName, processPid)
- resolutionCall = os.popen(cmd)
- results = resolutionCall.readlines()
- resolutionCall.close()
+ # raises an IOError if the command fails or isn't available
+ results = sysTools.call(cmd)
- if not results: raise IOError("Unable to resolve connections using: %s" % cmd)
+ if not results: raise IOError("No results found using: %s" % cmd)
# parses results for the resolution command
conn = []
@@ -139,31 +131,6 @@
RESOLVERS.append(r)
return r
-def _isAvailable(command, cached=True):
- """
- Checks the current PATH to see if a command is available or not. This returns
- True if an accessible executable by the name is found and False otherwise.
-
- Arguments:
- command - name of the command for which to search
- cached - this makes use of available cached results if true, otherwise
- they're overwritten
- """
-
- if cached and command in CMD_AVAILABLE_CACHE.keys():
- return CMD_AVAILABLE_CACHE[command]
- else:
- cmdExists = False
- for path in os.environ["PATH"].split(os.pathsep):
- cmdPath = os.path.join(path, command)
-
- if os.path.exists(cmdPath) and os.access(cmdPath, os.X_OK):
- cmdExists = True
- break
-
- CMD_AVAILABLE_CACHE[command] = cmdExists
- return cmdExists
-
if __name__ == '__main__':
# quick method for testing connection resolution
userInput = raw_input("Enter query (RESOLVER PROCESS_NAME [PID]: ").split()
@@ -263,8 +230,8 @@
# sets the default resolver to be the first found in the system's PATH
# (left as netstat if none are found)
for resolver in [CMD_NETSTAT, CMD_SS, CMD_LSOF]:
- if _isAvailable(CMD_STR[resolver]):
- self.defaultResolve = resolver
+ if sysTools.isAvailable(CMD_STR[resolver]):
+ self.defaultResolver = resolver
break
self._connections = [] # connection cache (latest results)
@@ -294,13 +261,16 @@
connResults = getConnections(resolver, self.processName, self.processPid)
lookupTime = time.time() - resolveStart
- log.log(log.DEBUG, "%s queried in %.4f seconds (%i results)" % (CMD_STR[resolver], lookupTime, len(connResults)))
-
self._connections = connResults
self.defaultRate = max(5, 10 % lookupTime)
if isDefault: self._subsiquentFailures = 0
except IOError, exc:
- log.log(log.INFO, str(exc)) # notice that a single resolution has failed
+ # this logs in a couple of cases:
+ # - special failures noted by getConnections (most cases are already
+ # logged via sysTools)
+ # - note failovers for default resolution methods
+ if str(exc).startswith("No results found using:"):
+ log.log(log.INFO, str(exc))
if isDefault:
self._subsiquentFailures += 1
Modified: arm/trunk/util/hostnames.py
===================================================================
--- arm/trunk/util/hostnames.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/hostnames.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -25,7 +25,6 @@
# - When adding/removing from the cache (prevents workers from updating
# an outdated cache reference).
-import os
import time
import socket
import threading
@@ -33,6 +32,8 @@
import Queue
import distutils.sysconfig
+import sysTools
+
RESOLVER = None # hostname resolver (service is stopped if None)
RESOLVER_LOCK = threading.RLock() # regulates assignment to the RESOLVER
RESOLVER_CACHE_SIZE = 700000 # threshold for when cached results are discarded
@@ -157,7 +158,7 @@
# get cache entry, raising if an exception and returning if a hostname
cacheRef = resolverRef.resolvedCache
- if ipAddr in cacheRef.keys():
+ if ipAddr in cacheRef:
entry = cacheRef[ipAddr][0]
if suppressIOExc and type(entry) == IOError: return None
elif isinstance(entry, Exception): raise entry
@@ -167,7 +168,7 @@
# if resolver has cached an IOError then flush the entry (this defaults to
# suppression since these error may be transient)
cacheRef = resolverRef.resolvedCache
- flush = ipAddr in cacheRef.keys() and type(cacheRef[ipAddr]) == IOError
+ flush = ipAddr in cacheRef and type(cacheRef[ipAddr]) == IOError
try: return resolverRef.getHostname(ipAddr, timeout, flush)
except IOError: return None
@@ -220,13 +221,8 @@
ipAddr - ip address to be resolved
"""
- hostCall = os.popen("host %s 2> /dev/null" % ipAddr)
- hostname = hostCall.read()
- hostCall.close()
+ hostname = sysTools.call("host %s" % ipAddr)[0].split()[-1:][0]
- if hostname: hostname = hostname.split()[-1:][0]
- else: raise IOError("lookup failed - is the host command available?")
-
if hostname == "reached":
# got message: ";; connection timed out; no servers could be reached"
raise IOError("lookup timed out")
@@ -291,7 +287,7 @@
# during this call)
cacheRef = self.resolvedCache
- if not flushCache and ipAddr in cacheRef.keys():
+ if not flushCache and ipAddr in cacheRef:
# cached response is available - raise if an error, return if a hostname
response = cacheRef[ipAddr][0]
if isinstance(response, Exception): raise response
@@ -307,7 +303,7 @@
startTime = time.time()
while timeout == None or time.time() - startTime < timeout:
- if ipAddr in cacheRef.keys():
+ if ipAddr in cacheRef:
# address was resolved - raise if an error, return if a hostname
response = cacheRef[ipAddr][0]
if isinstance(response, Exception): raise response
Modified: arm/trunk/util/panel.py
===================================================================
--- arm/trunk/util/panel.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/panel.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -17,7 +17,7 @@
FORMAT_TAGS = {"<b>": (_noOp, curses.A_BOLD),
"<u>": (_noOp, curses.A_UNDERLINE),
"<h>": (_noOp, curses.A_STANDOUT)}
-for colorLabel in uiTools.COLOR_LIST.keys(): FORMAT_TAGS["<%s>" % colorLabel] = (uiTools.getColor, colorLabel)
+for colorLabel in uiTools.COLOR_LIST: FORMAT_TAGS["<%s>" % colorLabel] = (uiTools.getColor, colorLabel)
class Panel():
"""
Added: arm/trunk/util/sysTools.py
===================================================================
--- arm/trunk/util/sysTools.py (rev 0)
+++ arm/trunk/util/sysTools.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -0,0 +1,156 @@
+"""
+Helper functions for working with the underlying system.
+"""
+
+import os
+import time
+import threading
+
+import log
+
+# mapping of commands to if they're available or not
+CMD_AVAILABLE_CACHE = {}
+
+# cached system call results, mapping the command issued to the (time, results) tuple
+CALL_CACHE = {}
+IS_FAILURES_CACHED = True # caches both successful and failed results if true
+CALL_CACHE_TRIM_SIZE = 600 # number of entries at which old results are trimmed
+CALL_CACHE_LOCK = threading.RLock() # governs concurrent modifications of CALL_CACHE
+
+def isAvailable(command, cached=True):
+ """
+ Checks the current PATH to see if a command is available or not. If a full
+ call is provided then this just checks the first command (for instance
+ "ls -a | grep foo" is truncated to "ls"). This returns True if an accessible
+ executable by the name is found and False otherwise.
+
+ Arguments:
+ command - command for which to search
+ cached - this makes use of available cached results if true, otherwise
+ they're overwritten
+ """
+
+ if " " in command: command = command.split(" ")[0]
+
+ if cached and command in CMD_AVAILABLE_CACHE:
+ return CMD_AVAILABLE_CACHE[command]
+ else:
+ cmdExists = False
+ for path in os.environ["PATH"].split(os.pathsep):
+ cmdPath = os.path.join(path, command)
+
+ if os.path.exists(cmdPath) and os.access(cmdPath, os.X_OK):
+ cmdExists = True
+ break
+
+ CMD_AVAILABLE_CACHE[command] = cmdExists
+ return cmdExists
+
+def call(command, cacheAge=0, suppressExc=False, quiet=True):
+ """
+ Convenience function for performing system calls, providing:
+ - suppression of any writing to stdout, both directing stderr to /dev/null
+ and checking for the existance of commands before executing them
+ - logging of results (command issued, runtime, success/failure, etc)
+ - optional exception suppression and caching (the max age for cached results
+ is a minute)
+
+ Arguments:
+ command - command to be issued
+ cacheAge - uses cached results rather than issuing a new request if last
+ fetched within this number of seconds (if zero then all
+ caching functionality is skipped)
+ suppressExc - provides None in cases of failure if True, otherwise IOErrors
+ are raised
+ quiet - if True, "2> /dev/null" is appended to all commands
+ """
+
+ # caching functionality (fetching and trimming)
+ if cacheAge > 0:
+ global CALL_CACHE, CALL_CACHE_TRIM_SIZE
+
+ # keeps consistancy that we never use entries over a minute old (these
+ # results are 'dirty' and might be trimmed at any time)
+ cacheAge = min(cacheAge, 60)
+
+ # if the cache is especially large then trim old entries
+ if len(CALL_CACHE) > CALL_CACHE_TRIM_SIZE:
+ CALL_CACHE_LOCK.acquire()
+
+ # checks that we haven't trimmed while waiting
+ if len(CALL_CACHE) > CALL_CACHE_TRIM_SIZE:
+ # constructs a new cache with only entries less than a minute old
+ newCache, currentTime = {}, time.time()
+
+ for cachedCommand, cachedResult in CALL_CACHE.items():
+ if currentTime - cachedResult[0] < 60:
+ newCache[cachedCommand] = cachedResult
+
+ # if the cache is almost as big as the trim size then we risk doing this
+ # frequently, so grow it and log
+ if len(newCache) > (0.75 * CALL_CACHE_TRIM_SIZE):
+ CALL_CACHE_TRIM_SIZE = len(newCache) * 2
+ log.log(log.INFO, "growing system call cache to %i entries" % CALL_CACHE_TRIM_SIZE)
+
+ CALL_CACHE = newCache
+ CALL_CACHE_LOCK.release()
+
+ # checks if we can make use of cached results
+ if command in CALL_CACHE and time.time() - CALL_CACHE[command][0] < cacheAge:
+ cachedResults = CALL_CACHE[command][1]
+ cacheAge = time.time() - CALL_CACHE[command][0]
+
+ if isinstance(cachedResults, IOError):
+ if IS_FAILURES_CACHED:
+ log.log(log.DEBUG, "system call (cached failure): %s (age: %0.1f seconds, error: %s)" % (command, cacheAge, str(cachedResults)))
+ if suppressExc: return None
+ else: raise cachedResults
+ else:
+ # flag was toggled after a failure was cached - reissue call, ignoring the cache
+ return call(command, 0, suppressExc, quiet)
+ else:
+ log.log(log.DEBUG, "system call (cached): %s (age: %0.1f seconds)" % (command, cacheAge))
+ return cachedResults
+
+ # preprocessing for the commands to prevent anything going to stdout
+ startTime = time.time()
+ commandComp = command.split("|")
+ commandCall, results, errorExc = None, None, None
+
+ for i in range(len(commandComp)):
+ subcommand = commandComp[i].strip()
+
+ if not isAvailable(subcommand): errorExc = IOError("'%s' is unavailable" % subcommand.split(" ")[0])
+ if quiet: commandComp[i] = "%s 2> /dev/null" % subcommand
+
+ # processes the system call
+ if not errorExc:
+ try:
+ commandCall = os.popen(" | ".join(commandComp))
+ results = commandCall.readlines()
+ except IOError, exc:
+ errorExc = exc
+
+ # make sure sys call is closed
+ if commandCall: commandCall.close()
+
+ if errorExc:
+ # log failure and either provide None or re-raise exception
+ log.log(log.INFO, "system call (failed): %s (error: %s)" % (command, str(errorExc)))
+ if cacheAge > 0 and IS_FAILURES_CACHED:
+ CALL_CACHE_LOCK.acquire()
+ CALL_CACHE[command] = (time.time(), errorExc)
+ CALL_CACHE_LOCK.release()
+
+ if suppressExc: return None
+ else: raise errorExc
+ else:
+ # log call information and if we're caching then save the results
+ log.log(log.DEBUG, "system call: %s (runtime: %0.2f seconds)" % (command, time.time() - startTime))
+ if cacheAge > 0:
+ CALL_CACHE_LOCK.acquire()
+ CALL_CACHE[command] = (time.time(), results)
+ CALL_CACHE_LOCK.release()
+
+ return results
+
Modified: arm/trunk/util/uiTools.py
===================================================================
--- arm/trunk/util/uiTools.py 2010-05-07 13:35:05 UTC (rev 22294)
+++ arm/trunk/util/uiTools.py 2010-05-07 16:25:49 UTC (rev 22295)
@@ -16,7 +16,7 @@
# mappings for getColor() - this uses the default terminal color scheme if
# color support is unavailable
COLOR_ATTR_INITIALIZED = False
-COLOR_ATTR = dict([(color, 0) for color in COLOR_LIST.keys()])
+COLOR_ATTR = dict([(color, 0) for color in COLOR_LIST])
# value tuples for label conversions (bytes / seconds, short label, long label)
SIZE_UNITS = [(1125899906842624.0, " PB", " Petabyte"), (1099511627776.0, " TB", " Terabyte"),
@@ -165,7 +165,7 @@
if hasColorSupport:
colorpair = 0
- for colorName in COLOR_LIST.keys():
+ for colorName in COLOR_LIST:
fgColor = COLOR_LIST[colorName]
bgColor = -1 # allows for default (possibly transparent) background
colorpair += 1