[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [arm/master] Use pyflakes and pep8 apis rather than shelling out



commit be09186f744d9fd335fb354d957e5e766e847709
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Thu Jan 2 08:29:15 2014 -0800

    Use pyflakes and pep8 apis rather than shelling out
    
    Bringing over stem's recent improvements to use pyflakes and pep8 directly
    rather than shelling out. This results in a 62.7% reduction in our test runtime
    (from 2.63 to 0.98 seconds on my system).
---
 arm/headerPanel.py |    4 +-
 run_tests.py       |  138 ++++++++++++++++++++++++++++++++--------------------
 2 files changed, 88 insertions(+), 54 deletions(-)

diff --git a/arm/headerPanel.py b/arm/headerPanel.py
index addde9d..00796a1 100644
--- a/arm/headerPanel.py
+++ b/arm/headerPanel.py
@@ -21,8 +21,6 @@ import threading
 
 import arm.util.tracker
 
-import stem
-
 from stem.control import State
 from stem.util import conf, log, str_tools
 
@@ -173,6 +171,8 @@ class HeaderPanel(panel.Panel, threading.Thread):
       #  torTools.getConn().init(controller)
       #  log.notice("Reconnected to Tor's control port")
       #  arm.popups.showMsg("Tor reconnected", 1)
+
+      pass
     else: isKeystrokeConsumed = False
 
     return isKeystrokeConsumed
diff --git a/run_tests.py b/run_tests.py
index 3ed5e25..13570bc 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -79,6 +79,34 @@ def clean_orphaned_pyc():
           os.remove(pyc_path)
 
 
+def is_pyflakes_available():
+  """
+  Checks if pyflakes is availalbe.
+
+  :returns: **True** if we can use pyflakes and **False** otherwise
+  """
+
+  try:
+    import pyflakes
+    return True
+  except ImportError:
+    return False
+
+
+def is_pep8_available():
+  """
+  Checks if pep8 is availalbe.
+
+  :returns: **True** if we can use pep8 and **False** otherwise
+  """
+
+  try:
+    import pep8
+    return True
+  except ImportError:
+    return False
+
+
 def get_stylistic_issues(paths):
   """
   Checks for stylistic issues that are an issue according to the parts of PEP8
@@ -94,26 +122,25 @@ def get_stylistic_issues(paths):
   :returns: **dict** of the form ``path => [(line_number, message)...]``
   """
 
-  ignored_issues = ','.join(CONFIG["pep8.ignore"])
   issues = {}
 
-  for path in paths:
-    pep8_output = stem.util.system.call(
-      "pep8 --ignore %s %s" % (ignored_issues, path),
-      ignore_exit_status = True,
-    )
+  if is_pep8_available():
+    import pep8
 
-    for line in pep8_output:
-      line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line)
+    class StyleReport(pep8.BaseReport):
+      def __init__(self, options):
+        super(StyleReport, self).__init__(options)
 
-      if line_match:
-        path, line, _, issue = line_match.groups()
+      def error(self, line_number, offset, text, check):
+        code = super(StyleReport, self).error(line_number, offset, text, check)
 
-        if path in CONFIG["pep8.blacklist"]:
-          continue
+        if code:
+          issues.setdefault(self.filename, []).append((offset + line_number, "%s %s" % (code, text)))
 
-        issues.setdefault(path, []).append((int(line), issue))
+    style_checker = pep8.StyleGuide(ignore = CONFIG["pep8.ignore"], reporter = StyleReport)
+    style_checker.check_files(filter(lambda path: path not in CONFIG["pep8.blacklist"], _python_files(paths)))
 
+  for path in paths:
     for file_path in _get_files_with_suffix(path):
       if file_path in CONFIG["pep8.blacklist"]:
         continue
@@ -121,21 +148,25 @@ def get_stylistic_issues(paths):
       with open(file_path) as f:
         file_contents = f.read()
 
-      lines, file_issues, prev_indent = file_contents.split("\n"), [], 0
+      lines, prev_indent = file_contents.split("\n"), 0
       is_block_comment = False
 
       for index, line in enumerate(lines):
         whitespace, content = re.match("^(\s*)(.*)$", line).groups()
 
+        # TODO: This does not check that block indentations are two spaces
+        # because differentiating source from string blocks ("""foo""") is more
+        # of a pita than I want to deal with right now.
+
         if '"""' in content:
           is_block_comment = not is_block_comment
 
         if "\t" in whitespace:
-          file_issues.append((index + 1, "indentation has a tab"))
+          issues.setdefault(file_path, []).append((index + 1, "indentation has a tab"))
         elif "\r" in content:
-          file_issues.append((index + 1, "contains a windows newline"))
+          issues.setdefault(file_path, []).append((index + 1, "contains a windows newline"))
         elif content != content.rstrip():
-          file_issues.append((index + 1, "line has trailing whitespace"))
+          issues.setdefault(file_path, []).append((index + 1, "line has trailing whitespace"))
         elif content.lstrip().startswith("except") and content.endswith(", exc:"):
           # Python 2.6 - 2.7 supports two forms for exceptions...
           #
@@ -145,10 +176,7 @@ def get_stylistic_issues(paths):
           # The former is the old method and no longer supported in python 3
           # going forward.
 
-          file_issues.append((index + 1, "except clause should use 'as', not comma"))
-
-      if file_issues:
-        issues[file_path] = file_issues
+          issues.setdefault(file_path, []).append((index + 1, "except clause should use 'as', not comma"))
 
   return issues
 
@@ -162,47 +190,47 @@ def get_pyflakes_issues(paths):
   :returns: dict of the form ``path => [(line_number, message)...]``
   """
 
-  pyflakes_ignore = {}
+  issues = {}
 
-  for line in CONFIG["pyflakes.ignore"]:
-    path, issue = line.split("=>")
-    pyflakes_ignore.setdefault(path.strip(), []).append(issue.strip())
+  if is_pyflakes_available():
+    import pyflakes.api
+    import pyflakes.reporter
 
-  def is_ignored(path, issue):
-    # Paths in pyflakes_ignore are relative, so we need to check to see if our
-    # path ends with any of them.
+    class Reporter(pyflakes.reporter.Reporter):
+      def __init__(self):
+        self._ignored_issues = {}
 
-    for ignore_path in pyflakes_ignore:
-      if path.endswith(ignore_path) and issue in pyflakes_ignore[ignore_path]:
-        return True
+        for line in CONFIG["pyflakes.ignore"]:
+          path, issue = line.split("=>")
+          self._ignored_issues.setdefault(path.strip(), []).append(issue.strip())
 
-    return False
+      def unexpectedError(self, filename, msg):
+        self._register_issue(filename, None, msg)
 
-  # Pyflakes issues are of the form...
-  #
-  #   FILE:LINE: ISSUE
-  #
-  # ... for instance...
-  #
-  #   stem/prereq.py:73: 'long_to_bytes' imported but unused
-  #   stem/control.py:957: undefined name 'entry'
+      def syntaxError(self, filename, msg, lineno, offset, text):
+        self._register_issue(filename, lineno, msg)
 
-  issues = {}
+      def flake(self, msg):
+        self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args)
 
-  for path in paths:
-    pyflakes_output = stem.util.system.call(
-      "pyflakes %s" % path,
-      ignore_exit_status = True,
-    )
+      def _is_ignored(self, path, issue):
+        # Paths in pyflakes_ignore are relative, so we need to check to see if our
+        # path ends with any of them.
+
+        for ignored_path, ignored_issues in self._ignored_issues.items():
+          if path.endswith(ignored_path) and issue in ignored_issues:
+            return True
 
-    for line in pyflakes_output:
-      line_match = re.match("^(.*):(\d+): (.*)$", line)
+        return False
 
-      if line_match:
-        path, line, issue = line_match.groups()
+      def _register_issue(self, path, line_number, issue):
+        if not self._is_ignored(path, issue):
+          issues.setdefault(path, []).append((line_number, issue))
 
-        if not is_ignored(path, issue):
-          issues.setdefault(path, []).append((int(line), issue))
+    reporter = Reporter()
+
+    for path in _python_files(paths):
+      pyflakes.api.checkPath(path, reporter)
 
   return issues
 
@@ -228,5 +256,11 @@ def _get_files_with_suffix(base_path, suffix = ".py"):
           yield os.path.join(root, filename)
 
 
+def _python_files(paths):
+  for path in paths:
+    for file_path in _get_files_with_suffix(path):
+      yield file_path
+
+
 if __name__ == '__main__':
   main()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits