Add MypyBear

Add a bear that runs mypy, a checker for PEP 484 type hints.

Closes coala#601

Author: encukou

.ci/deps.sh

@@ -90,6 +90,7 @@ for dep_version in "${dep_versions[@]}" ; do
   source .ci/env_variables.sh
 
   pip install pip -U
+  pip install -U setuptools
   pip install -r test-requirements.txt
   pip install -r requirements.txt
 done

bears/python/MypyBear.py

@@ -0,0 +1,139 @@
+from collections import namedtuple
+import textwrap
+import sys
+
+from coalib.bearlib.abstractions.Linter import linter
+from coalib.bears.requirements.PipRequirement import PipRequirement
+from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
+from coalib.results.Result import Result
+
+
+class FlagInfo(namedtuple('FlagInfo', 'arg doc inverse')):
+    """
+    Information about a command-line flag.
+
+    :param arg:
+        The argument to pass to enable the flag.
+    :param doc:
+        A help text for the flag.
+    :param inverse:
+        Set to ``True`` when the coala option is the inverse of
+        the subprocess one, for example coala's ``allow_untyped_calls``
+        is the inverse of mypy's ``--disallow-untyped-calls``.
+    """
+
+    def want_flag(self, value):
+        """
+        Check if the flag should be added to the argument list.
+
+        :param value: The configuration value.
+        :return:      The flag value, may be negated if the flag specifies so.
+        """
+
+        if self.inverse:
+            value = not value
+        return value
+
+
+FLAG_MAP = {
+    'allow_untyped_functions': FlagInfo(
+        arg='--disallow-untyped-defs',
+        doc='Allow defining functions without type annotations or with '
+            'incomplete type annotations.',
+        inverse=True),
+    'allow_untyped_calls': FlagInfo(
+        arg='--disallow-untyped-calls',
+        doc='Allow calling functions without type annotations from '
+            'typed functions.',
+        inverse=True),
+    'check_untyped_function_bodies': FlagInfo(
+        arg='--check-untyped-defs',
+        doc='Do not check the interior of functions without type annotations.',
+        inverse=False),
+    'strict_optional': FlagInfo(
+        arg='--strict-optional',
+        doc='Enable experimental strict checks related to Optional types. See '
+            '<http://mypy-lang.blogspot.com.es/2016/07/mypy-043-released.html>'
+            ' for an explanation.',
+        inverse=False),
+}
+
+
+def add_param_docs(param_map):
+    """
+    Append documentation from FLAG_MAP to a function's docstring.
+
+    :param param_map:
+        A mapping of argument names (strings) to FlagInfo objects.
+    :return:
+        A decorator that appends flag information to a function's docstring.
+    """
+    def decorator(func):
+        func.__doc__ = textwrap.dedent(func.__doc__) + '\n'.join(
+            ':param {}:\n{}'.format(name, textwrap.indent(arg.doc, '    '))
+            for name, arg in param_map.items())
+        return func
+    return decorator
+
+
+# Mypy generates messages in the format:
+#    blabla.py: note: In function "f":
+#    blabla.py:2: error: Unsupported operand types for ...
+# The "note" messages are only adding info coala should already know,
+# so discard those. We're only capturing the errors.
+@linter(executable=sys.executable,
+        prerequisite_check_command=(sys.executable, '-m', 'mypy', '-V'),
+        output_format="regex",
+        output_regex=r'(?P<filename>[^:]+):((?P<line>\d+):)? '
+                      '(?P<severity>error): (?P<message>.*)')
+class MypyBear:
+    """
+    Type-checks your Python files!
+
+    Checks optional static typing using the mypy tool.
+    See <http://mypy.readthedocs.io/en/latest/basics.html> for info on how to
+    add static typing.
+    """
+
+    LANGUAGES = {"Python", "Python 2", "Python 3"}
+    AUTHORS = {'Petr Viktorin'}
+    REQUIREMENTS = {PipRequirement('mypy-lang', '0.*')}
+    AUTHORS_EMAILS = {'encukou@gmail.com'}
+    LICENSE = 'AGPL-3.0'
+
+    # This detects typing errors, which is pretty unique -- it doesn't
+    # make sense to add a category for it.
+    CAN_DETECT = set()
+
+    @add_param_docs(FLAG_MAP)
+    def create_arguments(self, filename, file, config_file,
+                         language: str="Python 3",
+                         python_version: str=None,
+                         allow_untyped_functions: bool=True,
+                         allow_untyped_calls: bool=True,
+                         check_untyped_function_bodies: bool=False,
+                         strict_optional: bool=False):
+        """
+        :param language:
+            Set to ``Python`` or ``Python 3`` to check Python 3.x source.
+            Use ``Python 2`` for Python 2.x.
+        :param python_version:
+            Set the specific Python version, e.g. ``3.5``.
+        """
+        args = ['-m', 'mypy']
+        if language.lower() == 'python 2':
+            args.append('--py2')
+        elif language.lower() not in ('python 3', 'python'):
+            # Ideally, this would fail the check, but there's no good
+            # way to fail from create_arguments.
+            # See https://github.com/coala-analyzer/coala/issues/2573
+            self.err(
+                'Language needs to be "Python", "Python 2" or "Python 3". '
+                'Assuming Python 3.')
+        if python_version:
+            args.extend(['--python-version', python_version])
+        loc = locals()
+        args.extend(flag.arg for name, flag in FLAG_MAP.items()
+                    if flag.want_flag(loc[name]))
+        args.append(filename)
+        return args

requirements.txt

@@ -29,3 +29,4 @@ vulture==0.10.*
 nbformat>=4.*
 pyflakes==1.2.*  # Although we don't need this directly, solves a dep conflict
 scspell3k==2.*
+mypy-lang==0.4.*

tests/python/MypyBearTest.py

@@ -0,0 +1,124 @@
+from queue import Queue
+from textwrap import dedent
+
+from bears.python.MypyBear import MypyBear
+from tests.BearTestHelper import generate_skip_decorator
+from tests.LocalBearTestHelper import LocalBearTestHelper
+from coalib.settings.Section import Section
+from coalib.settings.Setting import Setting
+from coalib.results.Result import Result
+from coalib.results.RESULT_SEVERITY import RESULT_SEVERITY
+from coalib.misc.ContextManagers import prepare_file
+
+
+@generate_skip_decorator(MypyBear)
+class MypyBearTest(LocalBearTestHelper):
+
+    def setUp(self):
+        self.section = Section('name')
+        self.queue = Queue()
+        self.uut = MypyBear(self.section, self.queue)
+
+    def test_variable(self):
+        self.check_validity(self.uut,
+                            ["a = 1  # type: int"], valid=True)
+        self.check_validity(self.uut,
+                            ["a = 'abc'  # type: int"], valid=False)
+
+    def test_call_sum(self):
+        self.check_validity(self.uut,
+                            ["sum([1, 2, 3])"], valid=True)
+        self.check_validity(self.uut,
+                            ["sum(1, 2, 3)"], valid=False)
+
+    def test_py2(self):
+        self.check_validity(self.uut,
+                            ["print(123)"], valid=True)
+        self.check_validity(self.uut,
+                            ["print 123"], valid=False)
+        self.section.append(Setting('language', 'Python 2'))
+        self.check_validity(self.uut,
+                            ["print 123"], valid=True)
+
+    def test_py2_version(self):
+        self.check_validity(self.uut,
+                            ["print(123)"], valid=True)
+        self.check_validity(self.uut,
+                            ["print 123"], valid=False)
+        self.section.append(Setting('python_version', '2.7'))
+        self.check_validity(self.uut,
+                            ["print 123"], valid=True)
+
+    def test_bad_language(self):
+        self.section.append(Setting('language', 'Piet'))
+        self.check_validity(self.uut,
+                            ["1 + 1"], valid=True)
+        while not self.queue.empty():
+            message = self.queue.get()
+            msg = ('Language needs to be "Python", "Python 2" or '
+                   '"Python 3". Assuming Python 3.')
+            if message.message == msg:
+                break
+        else:
+            assert False, 'Message not found'
+
+    def test_check_untyped_function_bodies(self):
+        source = dedent("""
+            def foo():
+                return 1 + "abc"
+        """).splitlines()
+        self.check_validity(self.uut, source, valid=True)
+        self.section.append(Setting('check_untyped_function_bodies', 'true'))
+        self.check_validity(self.uut, source, valid=False)
+
+    def test_allow_untyped_functions(self):
+        source = dedent("""
+            def foo():
+                pass
+        """).splitlines()
+        self.check_validity(self.uut, source, valid=True)
+        self.section.append(Setting('allow_untyped_functions', 'false'))
+        self.check_validity(self.uut, source, valid=False)
+
+    def test_allow_untyped_calls(self):
+        source = dedent("""
+            def foo():
+                pass
+
+            foo()
+        """).splitlines()
+        self.check_validity(self.uut, source, valid=True)
+        self.section.append(Setting('allow_untyped_calls', 'false'))
+        self.check_validity(self.uut, source, valid=False)
+
+    def test_strict_optional(self):
+        source = dedent("""
+            from typing import Optional, List
+
+            def read_data_file(path: Optional[str]) -> List[str]:
+                with open(path) as f:  # Error
+                    return f.read().split(',')
+        """).splitlines()
+        self.check_validity(self.uut, source, valid=True)
+        self.section.append(Setting('strict_optional', 'true'))
+        self.check_validity(self.uut, source, valid=False)
+
+    def test_discarded_note(self):
+        source = dedent("""
+            def f() -> None:
+                1 + "a"
+        """).splitlines()
+        prepared = prepare_file(source, filename=None, create_tempfile=True)
+        with prepared as (file, fname):
+            results = [
+                Result.from_values(
+                    message=(
+                        'Unsupported operand types for + ("int" and "str")'),
+                    file=fname,
+                    line=3,
+                    origin=self.uut,
+                    severity=RESULT_SEVERITY.MAJOR,
+                )
+            ]
+            self.check_results(self.uut, source, results=results,
+                               filename=fname, create_tempfile=False)