[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Mypy static checks



commit d0ed7ecd7485bd5cfa587bcc5b35e1783bb03961
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Mon Apr 13 18:10:08 2020 -0700

    Mypy static checks
    
    Integrating Mypy (http://mypy-lang.org/) for static type checks. Presently it's
    reporting hundreds of issues so this will require some more work.
---
 run_tests.py            |   4 +-
 stem/util/test_tools.py | 112 +++++++++++++++++++++++++++++++++++++++---------
 test/task.py            |  15 ++++++-
 3 files changed, 107 insertions(+), 24 deletions(-)

diff --git a/run_tests.py b/run_tests.py
index c9196e18..2ea07dab 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -217,12 +217,14 @@ def main():
     test.task.CRYPTO_VERSION,
     test.task.PYFLAKES_VERSION,
     test.task.PYCODESTYLE_VERSION,
+    test.task.MYPY_VERSION,
     test.task.CLEAN_PYC,
     test.task.UNUSED_TESTS,
     test.task.IMPORT_TESTS,
     test.task.REMOVE_TOR_DATA_DIR if args.run_integ else None,
     test.task.PYFLAKES_TASK if not args.specific_test else None,
     test.task.PYCODESTYLE_TASK if not args.specific_test else None,
+    test.task.MYPY_TASK if not args.specific_test else None,
   )
 
   # Test logging. If '--log-file' is provided we log to that location,
@@ -334,7 +336,7 @@ def main():
 
   static_check_issues = {}
 
-  for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK):
+  for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK, test.task.MYPY_TASK):
     if not task.is_available and task.unavailable_msg:
       println(task.unavailable_msg, ERROR)
     else:
diff --git a/stem/util/test_tools.py b/stem/util/test_tools.py
index d5d0f842..80de447e 100644
--- a/stem/util/test_tools.py
+++ b/stem/util/test_tools.py
@@ -23,9 +23,11 @@ to match just against the prefix or suffix. For instance...
 
   is_pyflakes_available - checks if pyflakes is available
   is_pycodestyle_available - checks if pycodestyle is available
+  is_mypy_available - checks if mypy is available
 
   pyflakes_issues - static checks for problems via pyflakes
   stylistic_issues - checks for PEP8 and other stylistic issues
+  type_issues - checks for type problems
 """
 
 import collections
@@ -47,6 +49,7 @@ from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, Tuple,
 CONFIG = stem.util.conf.config_dict('test', {
   'pycodestyle.ignore': [],
   'pyflakes.ignore': [],
+  'mypy.ignore': [],
   'exclude_paths': [],
 })
 
@@ -353,6 +356,16 @@ def is_pycodestyle_available() -> bool:
   return hasattr(pycodestyle, 'BaseReport')
 
 
+def is_mypy_available() -> bool:
+  """
+  Checks if mypy is available.
+
+  :returns: **True** if we can use mypy and **False** otherwise
+  """
+
+  return _module_exists('mypy.api')
+
+
 def stylistic_issues(paths: Sequence[str], check_newlines: bool = False, check_exception_keyword: bool = False, prefer_single_quotes: bool = False) -> Mapping[str, 'stem.util.test_tools.Issue']:
   """
   Checks for stylistic issues that are an issue according to the parts of PEP8
@@ -541,27 +554,8 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.
       def flake(self, msg: str) -> None:
         self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args, None)
 
-      def _is_ignored(self, path: str, issue: str) -> bool:
-        # Paths in pyflakes_ignore are relative, so we need to check to see if our
-        # path ends with any of them.
-
-        for ignored_path, ignored_issues in self._ignored_issues.items():
-          if path.endswith(ignored_path):
-            if issue in ignored_issues:
-              return True
-
-            for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]:
-              if issue.startswith(prefix):
-                return True
-
-            for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]:
-              if issue.endswith(suffix):
-                return True
-
-        return False
-
-      def _register_issue(self, path: str, line_number: int, issue: str, line: int) -> None:
-        if not self._is_ignored(path, issue):
+      def _register_issue(self, path: str, line_number: int, issue: str, line: str) -> None:
+        if not _is_ignored(self._ignored_issues, path, issue):
           if path and line_number and not line:
             line = linecache.getline(path, line_number).strip()
 
@@ -575,6 +569,65 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.
   return issues
 
 
+def type_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.Issue']:
+  """
+  Performs type checks via mypy. False positives can be ignored via
+  'mypy.ignore' entries in our 'test' config. For instance...
+
+  ::
+
+    mypy.ignore stem/util/system.py => Incompatible types in assignment*
+
+  :param list paths: paths to search for problems
+
+  :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances
+  """
+
+  issues = {}
+
+  if is_mypy_available():
+    import mypy.api
+
+    ignored_issues = {}
+
+    for line in CONFIG['mypy.ignore']:
+      path, issue = line.split('=>')
+      ignored_issues.setdefault(path.strip(), []).append(issue.strip())
+
+    lines = mypy.api.run(paths)[0].splitlines()  # mypy returns (report, errors, exit_status)
+
+    for line in lines:
+      # example:
+      # stem/util/__init__.py:89: error: Incompatible return value type (got "Union[bytes, str]", expected "bytes")
+
+      if line.startswith('Found ') and line.endswith(' source files)'):
+        continue  # ex. "Found 1786 errors in 45 files (checked 49 source files)"
+      elif line.count(':') < 3:
+        raise ValueError('Failed to parse mypy line: %s' % line)
+
+      path, line_number, _, issue = line.split(':', 3)
+      issue = issue.strip()
+
+      if line_number.isdigit():
+        line_number = int(line_number)
+      else:
+        raise ValueError('Malformed line number on: %s' % line)
+
+      if _is_ignored(ignored_issues, path, issue):
+        continue
+
+      # skip getting code if there's too many reported issues
+
+      if len(lines) < 25:
+        line = linecache.getline(path, line_number).strip()
+      else:
+        line = ''
+
+      issues.setdefault(path, []).append(Issue(line_number, issue, line))
+
+  return issues
+
+
 def _module_exists(module_name: str) -> bool:
   """
   Checks if a module exists.
@@ -603,3 +656,20 @@ def _python_files(paths: Sequence[str]) -> Iterator[str]:
 
       if not skip:
         yield file_path
+
+
+def _is_ignored(config: Mapping[str, Sequence[str]], path: str, issue: str) -> bool:
+  for ignored_path, ignored_issues in config.items():
+    if path.endswith(ignored_path):
+      if issue in ignored_issues:
+        return True
+
+      for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]:
+        if issue.startswith(prefix):
+          return True
+
+      for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]:
+        if issue.endswith(suffix):
+          return True
+
+  return False
diff --git a/test/task.py b/test/task.py
index 939e263c..2366564c 100644
--- a/test/task.py
+++ b/test/task.py
@@ -16,12 +16,14 @@
   |- CRYPTO_VERSION - checks our version of cryptography
   |- PYFLAKES_VERSION - checks our version of pyflakes
   |- PYCODESTYLE_VERSION - checks our version of pycodestyle
+  |- MYPY_VERSION - checks our version of mypy
   |- CLEAN_PYC - removes any *.pyc without a corresponding *.py
   |- REMOVE_TOR_DATA_DIR - removes our tor data directory
   |- IMPORT_TESTS - ensure all test modules have been imported
   |- UNUSED_TESTS - checks to see if any tests are missing from our settings
   |- PYFLAKES_TASK - static checks
-  +- PYCODESTYLE_TASK - style checks
+  |- PYCODESTYLE_TASK - style checks
+  +- MYPY_TASK - type checks
 """
 
 import importlib
@@ -60,12 +62,12 @@ SRC_PATHS = [os.path.join(test.STEM_BASE, path) for path in (
   'cache_fallback_directories.py',
   'setup.py',
   'tor-prompt',
-  os.path.join('docs', 'republish.py'),
   os.path.join('docs', 'roles.py'),
 )]
 
 PYFLAKES_UNAVAILABLE = 'Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n  https://pypi.org/project/pyflakes/\n'
 PYCODESTYLE_UNAVAILABLE = 'Style checks require pycodestyle version 1.4.2 or later. Please install it from...\n  https://pypi.org/project/pycodestyle/\n'
+MYPY_UNAVAILABLE = 'Type checks require mypy. Please install it from...\n  http://mypy-lang.org/\n'
 
 
 def _check_stem_version():
@@ -324,6 +326,7 @@ PLATFORM_VERSION = Task('operating system', _check_platform_version)
 CRYPTO_VERSION = ModuleVersion('cryptography version', 'cryptography', lambda: test.require.CRYPTOGRAPHY_AVAILABLE)
 PYFLAKES_VERSION = ModuleVersion('pyflakes version', 'pyflakes')
 PYCODESTYLE_VERSION = ModuleVersion('pycodestyle version', ['pycodestyle', 'pep8'])
+MYPY_VERSION = ModuleVersion('mypy version', 'mypy.version')
 CLEAN_PYC = Task('checking for orphaned .pyc files', _clean_orphaned_pyc, (SRC_PATHS,), print_runtime = True)
 REMOVE_TOR_DATA_DIR = Task('emptying our tor data directory', _remove_tor_data_dir)
 IMPORT_TESTS = Task('importing test modules', _import_tests, print_runtime = True)
@@ -348,3 +351,11 @@ PYCODESTYLE_TASK = StaticCheckTask(
   is_available = stem.util.test_tools.is_pycodestyle_available(),
   unavailable_msg = PYCODESTYLE_UNAVAILABLE,
 )
+
+MYPY_TASK = StaticCheckTask(
+  'running mypy',
+  stem.util.test_tools.type_issues,
+  args = ([os.path.join(test.STEM_BASE, 'stem')],),
+  is_available = stem.util.test_tools.is_mypy_available(),
+  unavailable_msg = MYPY_UNAVAILABLE,
+)



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits