Linter highlighting

CHANGES.rst

@@ -186,8 +186,8 @@ Added
   against when finding out modified lines. Defaults to ``HEAD`` as before.
 - ``--no-skip-string-normalization`` flag to override
   ``skip_string_normalization = true`` from a configuration file
-- The ``--diff`` option will highlight syntax on screen if the ``pygments`` package is
-  available.
+- the ``--diff`` and ``--lint`` options will highlight syntax on screen if the
+  pygments_ package is available.
 
 Fixed
 -----
@@ -257,3 +257,4 @@ Added
 .. _Black 19.10: https://github.com/psf/black/blob/master/CHANGES.md#1910b0
 .. _Black 20.8: https://github.com/psf/black/blob/master/CHANGES.md#208b0
 .. _Pylint: https://pypi.org/project/pylint
+.. _pygments: https://pypi.org/project/Pygments/

mypy.ini

@@ -37,6 +37,10 @@ disallow_any_explicit = False
 [mypy-darker.config]
 disallow_subclassing_any = False
 
+[mypy-darker.highlighting.lexers]
+disallow_any_unimported = False
+disallow_subclassing_any = False
+
 [mypy-darker.tests.conftest]
 disallow_any_unimported = False
 

pyproject.toml

@@ -3,11 +3,7 @@
 requires = ["setuptools", "wheel"]  # PEP 508 specifications.
 
 [tool.isort]
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-line_length = 88
+profile = "black"
 known_third_party = ["pytest"]
 
 [tool.darker]

setup.cfg

@@ -38,6 +38,9 @@ where = src
 [options.entry_points]
 console_scripts =
     darker = darker.__main__:main_with_error_handling
+pygments.lexers =
+    lint_location = darker.highlighting.lexers:LocationLexer
+    lint_description = darker.highlighting.lexers:DescriptionLexer
 
 [options.extras_require]
 isort =

src/darker/__main__.py

@@ -30,6 +30,7 @@
     git_get_modified_files,
 )
 from darker.help import ISORT_INSTRUCTION
+from darker.highlighting import colorize
 from darker.import_sorting import apply_isort, isort
 from darker.linting import run_linters
 from darker.utils import GIT_DATEFORMAT, TextDocument, debug_dump, get_common_root
@@ -249,18 +250,7 @@ def print_diff(
             n=5,  # Black shows 5 lines of context, do the same
         )
     )
-
-    if sys.stdout.isatty():
-        try:
-            from pygments import highlight
-            from pygments.formatters import TerminalFormatter
-            from pygments.lexers import DiffLexer
-        except ImportError:
-            print(diff)
-        else:
-            print(highlight(diff, DiffLexer(), TerminalFormatter()))
-    else:
-        print(diff)
+    print(colorize(diff, "diff"))
 
 
 def print_source(new: TextDocument) -> None:

src/darker/highlighting/__init__.py

@@ -0,0 +1,12 @@
+"""Highlighting of terminal output"""
+
+try:
+    import pygments  # noqa: F401
+except ImportError:
+    from darker.highlighting import without_pygments
+
+    colorize = without_pygments.colorize
+else:
+    from darker.highlighting import with_pygments
+
+    colorize = with_pygments.colorize

src/darker/highlighting/lexers.py

@@ -0,0 +1,88 @@
+"""Custom Pygments lexers for highlighting linter output"""
+
+from typing import Generator, Tuple
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, combined
+from pygments.lexers.python import PythonLexer
+from pygments.token import Error, Number, String, Text, _TokenType
+
+
+class LocationLexer(Lexer):
+    """Lexer for linter output ``path:line:col:`` prefix"""
+
+    aliases = ["lint_location"]
+
+    def get_tokens_unprocessed(
+        self, text: str
+    ) -> Generator[Tuple[int, _TokenType, str], None, None]:
+        """Tokenize and generate (index, tokentype, value) tuples for highlighted tokens
+
+        "index" is the starting position of the token within the input text.
+
+        """
+        path, *positions = text.split(":")
+        yield 0, String, path
+        pos = len(path)
+        for position in positions:
+            yield pos, Text, ":"
+            yield pos + 1, Number, position
+            pos += 1 + len(position)
+
+
+class DescriptionLexer(RegexLexer):
+    """Lexer for linter output descriptions
+
+    Highlights embedded Python code and expressions using the Python 3 lexer.
+
+    """
+
+    aliases = "lint_description"
+
+    # Make normal text in linter messages look like strings in source code.
+    # This is a decent choice since it lets source code stand out fairly well.
+    message = String
+
+    # Customize the Python lexer
+    tokens = PythonLexer.tokens.copy()
+
+    # Move the main Python lexer into a separate state
+    tokens["python"] = tokens["root"]
+    tokens["python"].insert(0, ('"', message, "#pop"))
+    tokens["python"].insert(0, ("'", message, "#pop"))
+
+    # The root state manages a possible prefix for the description.
+    # It highlights error codes, and also catches coverage output and assumes that
+    # Python code follows and uses the Python lexer to highlight that.
+    tokens["root"] = [
+        (r"\s*no coverage: ", message, "python"),
+        (r"[CEFNW]\d{3,4}\b|error\b", Error, "description"),
+        (r"", Text, "description"),
+    ]
+
+    # Highlight a single space-separated word using the Python lexer
+    tokens["one-python-identifier"] = [
+        (" ", message, "#pop"),
+    ]
+
+    # The description state handles everything after the description prefix
+    tokens["description"] = [
+        # Highlight quoted expressions using the Python lexer.
+        ('"', message, combined("python", "dqs")),
+        ("'", message, combined("python", "sqs")),
+        # Also catch a few common patterns which are followed by Python expressions,
+        # but exclude a couple of special cases.
+        (r"\bUnused (argument|variable) ", message),
+        (
+            r"\b(Returning|Unused|Base type|imported from) ",
+            message,
+            combined("one-python-identifier", "python"),
+        ),
+        # Highlight parenthesized message identifiers at the end of messages
+        (
+            r"(\()([a-z][a-z-]+[a-z])(\))(\s*)$",
+            bygroups(message, Error, message, message),
+        ),
+        # Everything else is considered just plain non-highlighted text
+        (r"\s+", message),
+        (r"\S+", message),
+    ]

src/darker/highlighting/with_pygments.py

@@ -0,0 +1,20 @@
+"""Linter output highlighting helper to be used when Pygments is installed"""
+
+import sys
+from typing import cast
+
+from pygments import highlight
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.lexers import get_lexer_by_name
+
+
+def colorize(output: str, lexer_name: str) -> str:
+    """Return the output highlighted for terminal if Pygments is available"""
+    if not highlight or not sys.stdout.isatty():
+        return output
+    lexer = get_lexer_by_name(lexer_name)
+    highlighted = highlight(output, lexer, TerminalFormatter())
+    if "\n" not in output:
+        # see https://github.com/pygments/pygments/issues/1107
+        highlighted = highlighted.rstrip("\n")
+    return cast(str, highlighted)

src/darker/highlighting/without_pygments.py

@@ -0,0 +1,6 @@
+"""Linter output highlighting helper to be used when Pygments is not installed"""
+
+
+def colorize(output: str, lexer_name: str) -> str:  # pylint: disable=unused-argument
+    """Return the output unaltered"""
+    return output

src/darker/linting.py

@@ -23,25 +23,24 @@
 from contextlib import contextmanager
 from pathlib import Path
 from subprocess import PIPE, Popen
-from typing import IO, Generator, List, Set, Tuple, Union
+from typing import IO, Generator, List, Set, Tuple
 
 from darker.git import WORKTREE, EditedLinenumsDiffer, RevisionRange
+from darker.highlighting import colorize
 
 logger = logging.getLogger(__name__)
 
 
-def _parse_linter_line(
-    line: str, root: Path
-) -> Union[Tuple[Path, int], Tuple[None, None]]:
+def _parse_linter_line(line: str, root: Path) -> Tuple[Path, int, str, str]:
     # Parse an error/note line.
     # Given: line == "dir/file.py:123: error: Foo\n"
     # Sets: path = Path("abs/path/to/dir/file.py:123"
     #       linenum = 123
     #       description = "error: Foo"
     try:
-        location, _ = line[:-1].split(": ", 1)
-        path_str, linenum_bytes, *rest = location.split(":")
-        linenum = int(linenum_bytes)
+        location, description = line[:-1].split(": ", 1)
+        path_str, linenum_str, *rest = location.split(":")
+        linenum = int(linenum_str)
         if len(rest) > 1:
             raise ValueError("Too many colon-separated tokens")
         if len(rest) == 1:
@@ -53,10 +52,10 @@ def _parse_linter_line(
         # "Found XX errors in YY files (checked ZZ source files)"
         # "Success: no issues found in 1 source file"
         logger.debug("Unparseable linter output: %s", line[:-1])
-        return None, None
+        return Path(), 0, "", ""
     path_from_cwd = Path(path_str).absolute()
     path_in_repo = path_from_cwd.relative_to(root)
-    return path_in_repo, linenum
+    return path_in_repo, linenum, location + ":", description
 
 
 def _require_rev2_worktree(rev2: str) -> None:
@@ -117,8 +116,17 @@ def run_linter(
     with _check_linter_output(cmdline, root, paths) as linter_stdout:
         prev_path, prev_linenum = None, 0
         for line in linter_stdout:
-            path_in_repo, linter_error_linenum = _parse_linter_line(line, root)
-            if path_in_repo is None or path_in_repo in missing_files:
+            (
+                path_in_repo,
+                linter_error_linenum,
+                location,
+                description,
+            ) = _parse_linter_line(line, root)
+            if (
+                path_in_repo is None
+                or path_in_repo in missing_files
+                or linter_error_linenum == 0
+            ):
                 continue
             try:
                 edited_linenums = edited_linenums_differ.compare_revisions(
@@ -132,7 +140,8 @@ def run_linter(
                 if path_in_repo != prev_path or linter_error_linenum > prev_linenum + 1:
                     print()
                 prev_path, prev_linenum = path_in_repo, linter_error_linenum
-                print(line, end="")
+                print(colorize(location, "lint_location"), end=" ")
+                print(colorize(description, "lint_description"))
                 error_count += 1
     return error_count