[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [arm/master] Using stem's proc util



commit 75ed2eef3750eaa8d6d8ba56c55b545e5a5b8053
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Wed Jan 2 18:46:24 2013 -0800

    Using stem's proc util
    
    Stem's proc util is a direct counterpart for arm's. The only big difference is
    of course the tests. :)
---
 src/util/__init__.py    |    2 +-
 src/util/connections.py |    8 +-
 src/util/procTools.py   |  303 -----------------------------------------------
 src/util/sysTools.py    |   20 ++--
 src/util/torTools.py    |   14 +-
 5 files changed, 22 insertions(+), 325 deletions(-)

diff --git a/src/util/__init__.py b/src/util/__init__.py
index 9834bb1..16b28ee 100644
--- a/src/util/__init__.py
+++ b/src/util/__init__.py
@@ -4,5 +4,5 @@ application's status, making cross platform system calls, parsing tor data,
 and safely working with curses (hiding some of the gory details).
 """
 
-__all__ = ["connections", "hostnames", "panel", "procTools", "procName", "sysTools", "textInput", "torConfig", "torTools", "uiTools"]
+__all__ = ["connections", "hostnames", "panel", "procName", "sysTools", "textInput", "torConfig", "torTools", "uiTools"]
 
diff --git a/src/util/connections.py b/src/util/connections.py
index 5c66c1d..d633c03 100644
--- a/src/util/connections.py
+++ b/src/util/connections.py
@@ -21,9 +21,9 @@ import os
 import time
 import threading
 
-from util import procTools, sysTools
+from util import sysTools
 
-from stem.util import conf, enum, log
+from stem.util import conf, enum, log, proc
 
 # enums for connection resolution utilities
 Resolver = enum.Enum(("PROC", "proc"),
@@ -226,7 +226,7 @@ def getConnections(resolutionCmd, processName, processPid = ""):
       raise ValueError("proc resolution requires a pid")
     
     try:
-      return procTools.getConnections(processPid)
+      return proc.get_connections(processPid)
     except Exception, exc:
       raise IOError(str(exc))
   else:
@@ -337,7 +337,7 @@ def getSystemResolvers(osType = None):
     resolvers = [Resolver.NETSTAT, Resolver.SOCKSTAT, Resolver.LSOF, Resolver.SS]
   
   # proc resolution, by far, outperforms the others so defaults to this is able
-  if procTools.isProcAvailable():
+  if proc.is_available():
     resolvers = [Resolver.PROC] + resolvers
   
   return resolvers
diff --git a/src/util/procTools.py b/src/util/procTools.py
deleted file mode 100644
index 8a895fe..0000000
--- a/src/util/procTools.py
+++ /dev/null
@@ -1,303 +0,0 @@
-"""
-Helper functions for querying process and system information from the /proc
-contents. Fetching information this way provides huge performance benefits
-over lookups via system utilities (ps, netstat, etc). For instance, resolving
-connections this way cuts the runtime by around 90% verses the alternatives.
-These functions may not work on all platforms (only Linux?).
-
-All functions raise IOErrors if unable to read their respective proc files.
-
-The method for reading these files (and some of the code) are borrowed from
-psutil:
-https://code.google.com/p/psutil/
-which was written by Jay Loden, Dave Daeschler, Giampaolo Rodola' and is under
-the BSD license.
-"""
-
-import os
-import sys
-import time
-import socket
-import base64
-
-from stem.util import conf, enum, log
-
-# cached system values
-SYS_START_TIME, SYS_PHYSICAL_MEMORY = None, None
-CLOCK_TICKS = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
-Stat = enum.Enum("COMMAND", "CPU_UTIME", "CPU_STIME", "START_TIME")
-
-CONFIG = conf.config_dict("arm", {
-  "queries.useProc": True,
-})
-
-def isProcAvailable():
-  """
-  Provides true if configured to use proc resolution and it's available on the
-  platform, false otherwise.
-  """
-  
-  return CONFIG["queries.useProc"] and os.uname()[0] == "Linux"
-
-def getSystemStartTime():
-  """
-  Provides the unix time (seconds since epoch) when the system started.
-  """
-  
-  global SYS_START_TIME
-  if not SYS_START_TIME:
-    startTime = time.time()
-    statFile = open('/proc/stat')
-    statLines = statFile.readlines()
-    statFile.close()
-    
-    for line in statLines:
-      if line.startswith('btime'):
-        SYS_START_TIME = float(line.strip().split()[1])
-        break
-    
-    _logProcRuntime("system start time", "/proc/stat[btime]", startTime)
-  
-  return SYS_START_TIME
-
-def getPhysicalMemory():
-  """
-  Provides the total physical memory on the system in bytes.
-  """
-  
-  global SYS_PHYSICAL_MEMORY
-  if not SYS_PHYSICAL_MEMORY:
-    startTime = time.time()
-    memFile = open('/proc/meminfo')
-    memLines = memFile.readlines()
-    memFile.close()
-    
-    for line in memLines:
-      if line.startswith('MemTotal:'):
-        SYS_PHYSICAL_MEMORY = int(line.split()[1]) * 1024
-    
-    _logProcRuntime("system physical memory", "/proc/meminfo[MemTotal]", startTime)
-  
-  return SYS_PHYSICAL_MEMORY
-
-def getPwd(pid):
-  """
-  Provides the current working directory for the given process.
-  
-  Arguments:
-    pid - queried process
-  """
-  
-  startTime = time.time()
-  if pid == 0: cwd = ""
-  else: cwd = os.readlink("/proc/%s/cwd" % pid)
-  _logProcRuntime("cwd", "/proc/%s/cwd" % pid, startTime)
-  return cwd
-
-def getUid(pid):
-  """
-  Provides the user ID the given process is running under. This is None if it
-  can't be determined.
-  
-  Arguments:
-    pid - queried process
-  """
-  
-  startTime = time.time()
-  statusFile = open("/proc/%s/status" % pid)
-  statusFileLines = statusFile.readlines()
-  statusFile.close()
-  
-  result = None
-  for line in statusFileLines:
-    if line.startswith("Uid:"):
-      lineComp = line.split()
-      
-      if len(lineComp) >= 2 and lineComp[1].isdigit():
-        result = lineComp[1]
-  
-  _logProcRuntime("uid", "/proc/%s/status[Uid]" % pid, startTime)
-  return result
-
-def getMemoryUsage(pid):
-  """
-  Provides the memory usage in bytes for the given process of the form:
-  (residentSize, virtualSize)
-  
-  Arguments:
-    pid - queried process
-  """
-  
-  # checks if this is the kernel process
-  if pid == 0: return (0, 0)
-  
-  startTime = time.time()
-  statusFile = open("/proc/%s/status" % pid)
-  statusFileLines = statusFile.readlines()
-  statusFile.close()
-  
-  residentSize, virtualSize = None, None
-  for line in statusFileLines:
-    if line.startswith("VmRSS"):
-      residentSize = int(line.split()[1]) * 1024
-      if virtualSize != None: break
-    elif line.startswith("VmSize:"):
-      virtualSize = int(line.split()[1]) * 1024
-      if residentSize != None: break
-  
-  _logProcRuntime("memory usage", "/proc/%s/status[VmRSS|VmSize]" % pid, startTime)
-  return (residentSize, virtualSize)
-
-def getStats(pid, *statTypes):
-  """
-  Provides process specific information. Options are:
-  Stat.COMMAND      command name under which the process is running
-  Stat.CPU_UTIME    total user time spent on the process
-  Stat.CPU_STIME    total system time spent on the process
-  Stat.START_TIME   when this process began, in unix time
-  
-  Arguments:
-    pid       - queried process
-    statTypes - information to be provided back
-  """
-  
-  startTime = time.time()
-  statFilePath = "/proc/%s/stat" % pid
-  statFile = open(statFilePath)
-  statContents = statFile.read().strip()
-  statFile.close()
-  
-  # contents are of the form:
-  # 8438 (tor) S 8407 8438 8407 34818 8438 4202496...
-  statComp = []
-  cmdStart, cmdEnd = statContents.find("("), statContents.find(")")
-  
-  if cmdStart != -1 and cmdEnd != -1:
-    statComp.append(statContents[:cmdStart])
-    statComp.append(statContents[cmdStart + 1:cmdEnd])
-    statComp += statContents[cmdEnd + 1:].split()
-  
-  if len(statComp) != 44:
-    raise IOError("stat file had an unexpected format: %s" % statFilePath)
-  
-  results, queriedStats = [], []
-  for statType in statTypes:
-    if statType == Stat.COMMAND:
-      queriedStats.append("command")
-      if pid == 0: results.append("sched")
-      else: results.append(statComp[1])
-    elif statType == Stat.CPU_UTIME:
-      queriedStats.append("utime")
-      if pid == 0: results.append("0")
-      else: results.append(str(float(statComp[13]) / CLOCK_TICKS))
-    elif statType == Stat.CPU_STIME:
-      queriedStats.append("stime")
-      if pid == 0: results.append("0")
-      else: results.append(str(float(statComp[14]) / CLOCK_TICKS))
-    elif statType == Stat.START_TIME:
-      queriedStats.append("start time")
-      if pid == 0: return getSystemStartTime()
-      else:
-        # According to documentation, starttime is in field 21 and the unit is
-        # jiffies (clock ticks). We divide it for clock ticks, then add the
-        # uptime to get the seconds since the epoch.
-        pStartTime = float(statComp[21]) / CLOCK_TICKS
-        results.append(str(pStartTime + getSystemStartTime()))
-  
-  _logProcRuntime("process %s" % ", ".join(queriedStats), "/proc/%s/stat" % pid, startTime)
-  return results
-
-def getConnections(pid):
-  """
-  Provides a listing of connection tuples of the form:
-  [(local_ipAddr1, local_port1, foreign_ipAddr1, foreign_port1), ...]
-  
-  If the information about a connection can't be queried (often due to
-  permission issues) then it's excluded from the listing.
-  
-  Arguments:
-    pid - ID of the process to be resolved
-  """
-  
-  if pid == "0": return []
-  
-  # fetches the inode numbers for socket file descriptors
-  startTime = time.time()
-  inodes = []
-  for fd in os.listdir("/proc/%s/fd" % pid):
-    try:
-      # File descriptor link, such as 'socket:[30899]'
-      fdName = os.readlink("/proc/%s/fd/%s" % (pid, fd))
-      
-      if fdName.startswith('socket:['):
-        inodes.append(fdName[8:-1])
-    except OSError:
-      pass # most likely couldn't be read due to permissions
-  
-  if not inodes:
-    # unable to fetch any connections for this process
-    return []
-  
-  # check for the connection information from the /proc/net contents
-  conn = []
-  for procFilePath in ("/proc/net/tcp", "/proc/net/udp"):
-    procFile = open(procFilePath)
-    procFile.readline() # skip the first line
-    
-    for line in procFile:
-      _, lAddr, fAddr, status, _, _, _, _, _, inode = line.split()[:10]
-      
-      if inode in inodes:
-        # if a tcp connection, skip if it isn't yet established
-        if procFilePath.endswith("/tcp") and status != "01":
-          continue
-        
-        localIp, localPort = _decodeProcAddressEncoding(lAddr)
-        foreignIp, foreignPort = _decodeProcAddressEncoding(fAddr)
-        conn.append((localIp, localPort, foreignIp, foreignPort))
-    
-    procFile.close()
-  
-  _logProcRuntime("process connections", "/proc/net/[tcp|udp]", startTime)
-  
-  return conn
-
-def _decodeProcAddressEncoding(addr):
-  """
-  Translates an address entry in the /proc/net/* contents to a human readable
-  form, for instance:
-  "0500000A:0016" -> ("10.0.0.5", "22")
-  
-  Reference:
-  http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html
-  
-  Arguments:
-    addr - proc address entry to be decoded
-  """
-  
-  ip, port = addr.split(':')
-  
-  # the port is represented as a two-byte hexadecimal number
-  port = str(int(port, 16))
-  
-  if sys.version_info >= (3,):
-    ip = ip.encode('ascii')
-  
-  # The IPv4 address portion is a little-endian four-byte hexadecimal number.
-  # That is, the least significant byte is listed first, so we need to reverse
-  # the order of the bytes to convert it to an IP address.
-  #
-  # This needs to account for the endian ordering as per...
-  # http://code.google.com/p/psutil/issues/detail?id=201
-  # https://trac.torproject.org/projects/tor/ticket/4777
-  
-  if sys.byteorder == 'little':
-    ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)[::-1])
-  else:
-    ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip))
-  
-  return (ip, port)
-
-def _logProcRuntime(parameter, procLocation, startTime):
-  log.debug("proc call (%s): %s (runtime: %0.4f)" % (parameter, procLocation, time.time() - startTime))
-
diff --git a/src/util/sysTools.py b/src/util/sysTools.py
index 48bb9ed..2be73fd 100644
--- a/src/util/sysTools.py
+++ b/src/util/sysTools.py
@@ -6,9 +6,9 @@ import os
 import time
 import threading
 
-from util import procTools, uiTools
+from util import uiTools
 
-from stem.util import conf, log
+from stem.util import conf, log, proc
 
 # Mapping of commands to if they're available or not. This isn't always
 # reliable, failing for some special commands. For these the cache is
@@ -120,9 +120,9 @@ def getProcessName(pid, default = None, cacheFailure = True):
   processName, raisedExc = "", None
   
   # fetch it from proc contents if available
-  if procTools.isProcAvailable():
+  if proc.is_available():
     try:
-      processName = procTools.getStats(pid, procTools.Stat.COMMAND)[0]
+      processName = proc.get_stats(pid, proc.Stat.COMMAND)[0]
     except IOError, exc:
       raisedExc = exc
   
@@ -163,9 +163,9 @@ def getPwd(pid):
   elif pid in PWD_CACHE: return PWD_CACHE[pid]
   
   # try fetching via the proc contents if available
-  if procTools.isProcAvailable():
+  if proc.is_available():
     try:
-      pwd = procTools.getPwd(pid)
+      pwd = proc.get_cwd(pid)
       PWD_CACHE[pid] = pwd
       return pwd
     except IOError: pass # fall back to pwdx
@@ -403,7 +403,7 @@ class ResourceTracker(threading.Thread):
     self.memUsagePercentage = 0.0 # percentage cpu usage
     
     # resolves usage via proc results if true, ps otherwise
-    self._useProc = procTools.isProcAvailable()
+    self._useProc = proc.is_available()
     
     # used to get the deltas when querying cpu time
     self._lastCpuTotal = 0
@@ -469,15 +469,15 @@ class ResourceTracker(threading.Thread):
       newValues = {}
       try:
         if self._useProc:
-          utime, stime, startTime = procTools.getStats(self.processPid, procTools.Stat.CPU_UTIME, procTools.Stat.CPU_STIME, procTools.Stat.START_TIME)
+          utime, stime, startTime = proc.get_stats(self.processPid, proc.Stat.CPU_UTIME, proc.Stat.CPU_STIME, proc.Stat.START_TIME)
           totalCpuTime = float(utime) + float(stime)
           cpuDelta = totalCpuTime - self._lastCpuTotal
           newValues["cpuSampling"] = cpuDelta / timeSinceReset
           newValues["cpuAvg"] = totalCpuTime / (time.time() - float(startTime))
           newValues["_lastCpuTotal"] = totalCpuTime
           
-          memUsage = int(procTools.getMemoryUsage(self.processPid)[0])
-          totalMemory = procTools.getPhysicalMemory()
+          memUsage = int(proc.get_memory_usage(self.processPid)[0])
+          totalMemory = proc.get_physical_memory()
           newValues["memUsage"] = memUsage
           newValues["memUsagePercentage"] = float(memUsage) / totalMemory
         else:
diff --git a/src/util/torTools.py b/src/util/torTools.py
index 0ad914b..bce3481 100644
--- a/src/util/torTools.py
+++ b/src/util/torTools.py
@@ -16,9 +16,9 @@ import stem
 import stem.control
 import stem.descriptor
 
-from util import connections, procTools, sysTools, uiTools
+from util import connections, sysTools, uiTools
 
-from stem.util import conf, enum, log
+from stem.util import conf, enum, log, proc
 
 # enums for tor's controller state:
 # INIT - attached to a new controller
@@ -737,7 +737,7 @@ class Controller:
     self.connLock.acquire()
     
     result = None
-    if self.isAlive() and procTools.isProcAvailable():
+    if self.isAlive() and proc.is_available():
       myPid = self.getMyPid()
       
       if myPid:
@@ -1640,9 +1640,9 @@ class Controller:
           if myPid:
             # if proc contents are available then fetch the pid from there and
             # convert it to the username
-            if procTools.isProcAvailable():
+            if proc.is_available():
               try:
-                myUid = procTools.getUid(myPid)
+                myUid = proc.get_uid(myPid)
                 if myUid and myUid.isdigit():
                   result = pwd.getpwuid(int(myUid)).pw_name
               except: pass
@@ -1709,9 +1709,9 @@ class Controller:
         myPid = self.getMyPid()
         
         if myPid:
-          if procTools.isProcAvailable():
+          if proc.is_available():
             try:
-              result = float(procTools.getStats(myPid, procTools.Stat.START_TIME)[0])
+              result = float(proc.get_stats(myPid, proc.Stat.START_TIME)[0])
             except: pass
           
           if not result:



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits