From e7f8ba8392a9af93aa4841f363c0ce7632d7c5d9 Mon Sep 17 00:00:00 2001
From: Antti Kaihola <13725+akaihola@users.noreply.github.com>
Date: Tue, 29 Dec 2020 18:17:57 +0200
Subject: [PATCH] Change linting highlighting implementation a bit

- split out code into `darker.highlighting.*` modules
- register custom lexers as setuptools plugin entry points
- refer to built-in and custom Pygments lexers with names instead of
  object instances
- use `PythonLexer` directly instead of using the legacy
  `Python3Lexer` backwards-compatibility name
- separate dummy and real `colorize()` implementations (for use
  without and with Pygments, respectively)
- no need for "fake Pygments" module
- more unit tests
---
 mypy.ini                                      | 11 ++--
 setup.cfg                                     |  3 +
 src/darker/fake_pygments.py                   | 65 -------------------
 src/darker/highlighting/__init__.py           | 12 ++++
 .../lexers.py}                                | 60 ++++-------------
 src/darker/highlighting/with_pygments.py      | 20 ++++++
 src/darker/highlighting/without_pygments.py   |  6 ++
 src/darker/linting.py                         |  8 +--
 src/darker/tests/test_highlighting.py         | 64 +++++++++++++++---
 9 files changed, 116 insertions(+), 133 deletions(-)
 delete mode 100644 src/darker/fake_pygments.py
 create mode 100644 src/darker/highlighting/__init__.py
 rename src/darker/{highlighting.py => highlighting/lexers.py} (65%)
 create mode 100644 src/darker/highlighting/with_pygments.py
 create mode 100644 src/darker/highlighting/without_pygments.py

diff --git a/mypy.ini b/mypy.ini
index 203d30bd0..221ad7e0c 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -34,6 +34,10 @@ disallow_any_explicit = False
 [mypy-darker.config]
 disallow_subclassing_any = False
 
+[mypy-darker.highlighting.lexers]
+disallow_any_unimported = False
+disallow_subclassing_any = False
+
 [mypy-darker.tests.conftest]
 disallow_any_unimported = False
 
@@ -44,13 +48,6 @@ disallow_any_explicit = False
 disallow_any_decorated = False
 disallow_untyped_defs = False
 
-[mypy-darker.fake_pygments]
-disallow_any_explicit = False
-
-[mypy-darker.highlighting]
-disallow_any_unimported = False
-disallow_subclassing_any = False
-
 [mypy-darker.utils]
 warn_return_any = False
 
diff --git a/setup.cfg b/setup.cfg
index 5c694771f..5ae9520d5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,6 +36,9 @@ where = src
 [options.entry_points]
 console_scripts =
     darker = darker.__main__:main
+pygments.lexers =
+    lint_location = darker.highlighting.lexers:LocationLexer
+    lint_description = darker.highlighting.lexers:DescriptionLexer
 
 [options.extras_require]
 isort =
diff --git a/src/darker/fake_pygments.py b/src/darker/fake_pygments.py
deleted file mode 100644
index be42a6eef..000000000
--- a/src/darker/fake_pygments.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""Dummy implementation of Pygments parts we use. Used to satisfy Mypy."""
-
-# pylint: disable=too-few-public-methods
-
-from typing import IO, Any, Callable, Dict, Generator, List, Match, Tuple, Union
-
-
-class TerminalFormatter:
-    """Dummy replacement to satisfy Mypy"""
-
-
-class _TokenType:
-    """Dummy replacement to satisfy Mypy"""
-
-
-class Lexer:
-    """Dummy replacement to satisfy Mypy"""
-
-
-class RegexLexer(Lexer):
-    """Dummy replacement to satisfy Mypy"""
-
-
-TokenWithoutState = Tuple[str, _TokenType]
-TokenWithState = Tuple[str, _TokenType, str]
-Token = Union[TokenWithoutState, TokenWithState]
-
-
-class Python3Lexer(RegexLexer):
-    """Dummy replacement to satisfy Mypy"""
-
-    tokens: Dict[str, List[Token]] = {"root": []}
-
-
-class LexerContext:
-    """Dummy replacement to satisfy Mypy"""
-
-
-class combined(tuple):  # type: ignore  # pylint: disable=invalid-name
-    """Dummy implementation to satisfy Mypy"""
-
-
-# The `Any` below should really be a cyclic reference to `LexerCallback`,
-# but Mypy doesn't yet support that.
-LexerGenerator = Generator[
-    Tuple[int, Union[None, _TokenType, Any], LexerContext], None, None
-]
-LexerCallback = Callable[[Lexer, Match[str], LexerContext], LexerGenerator]
-
-
-def bygroups(*args: _TokenType) -> LexerCallback:
-    """Dummy implementation to satisfy Mypy"""
-
-
-def get_lexer_by_name(_alias: str, *options: Union[bool, int, str]) -> Lexer:
-    """Dummy implementation to satisfy Mypy"""
-
-
-def highlight(
-    code: str, lexer: Lexer, formatter: TerminalFormatter, outfile: IO[str] = None
-) -> str:
-    """Dummy implementation to satisfy Mypy"""
-
-
-Error = Number = String = Text = _TokenType()
diff --git a/src/darker/highlighting/__init__.py b/src/darker/highlighting/__init__.py
new file mode 100644
index 000000000..8b6d39327
--- /dev/null
+++ b/src/darker/highlighting/__init__.py
@@ -0,0 +1,12 @@
+"""Highlighting of terminal output"""
+
+try:
+    import pygments  # noqa: F401
+except ImportError:
+    from darker.highlighting import without_pygments
+
+    colorize = without_pygments.colorize
+else:
+    from darker.highlighting import with_pygments
+
+    colorize = with_pygments.colorize
diff --git a/src/darker/highlighting.py b/src/darker/highlighting/lexers.py
similarity index 65%
rename from src/darker/highlighting.py
rename to src/darker/highlighting/lexers.py
index 9f19198c7..c9212050d 100644
--- a/src/darker/highlighting.py
+++ b/src/darker/highlighting/lexers.py
@@ -1,52 +1,16 @@
-"""Highlighting of terminal output"""
-
-import sys
-from typing import Generator, Tuple, Union, cast
-
-try:
-    from pygments import highlight
-    from pygments.formatters.terminal import TerminalFormatter
-    from pygments.lexer import Lexer, RegexLexer, bygroups, combined
-    from pygments.lexers import get_lexer_by_name
-    from pygments.lexers.python import Python3Lexer
-    from pygments.token import Error, Number, String, Text, _TokenType
-
-    HAS_PYGMENTS = True
-except ImportError:
-    HAS_PYGMENTS = False
-
-    from darker.fake_pygments import (
-        Error,
-        Lexer,
-        Number,
-        Python3Lexer,
-        RegexLexer,
-        String,
-        TerminalFormatter,
-        Text,
-        _TokenType,
-        bygroups,
-        combined,
-        get_lexer_by_name,
-        highlight,
-    )
-
-
-def colorize(output: str, lexer: Union[str, Lexer]) -> str:
-    """Return the output highlighted for terminal if Pygments is available"""
-    if not HAS_PYGMENTS or not sys.stdout.isatty():
-        return output
-    if isinstance(lexer, str):
-        lexer = get_lexer_by_name(lexer)
-    highlighted = highlight(output, lexer, TerminalFormatter())
-    if "\n" not in output:
-        # see https://github.com/pygments/pygments/issues/1107
-        highlighted = highlighted.rstrip("\n")
-    return cast(str, highlighted)
+"""Custom Pygments lexers for highlighting linter output"""
+
+from typing import Generator, Tuple
+
+from pygments.lexer import Lexer, RegexLexer, bygroups, combined
+from pygments.lexers.python import PythonLexer
+from pygments.token import Error, Number, String, Text, _TokenType
 
 
 class LocationLexer(Lexer):
-    """Lexer for linter output ``path:line:col:` prefix"""
+    """Lexer for linter output ``path:line:col:`` prefix"""
+
+    aliases = ["lint_location"]
 
     def get_tokens_unprocessed(
         self, text: str
@@ -72,12 +36,14 @@ class DescriptionLexer(RegexLexer):
 
     """
 
+    aliases = "lint_description"
+
     # Make normal text in linter messages look like strings in source code.
     # This is a decent choice since it lets source code stand out fairly well.
     message = String
 
     # Customize the Python lexer
-    tokens = Python3Lexer.tokens.copy()
+    tokens = PythonLexer.tokens.copy()
 
     # Move the main Python lexer into a separate state
     tokens["python"] = tokens["root"]
diff --git a/src/darker/highlighting/with_pygments.py b/src/darker/highlighting/with_pygments.py
new file mode 100644
index 000000000..d25f42244
--- /dev/null
+++ b/src/darker/highlighting/with_pygments.py
@@ -0,0 +1,20 @@
+"""Linter output highlighting helper to be used when Pygments is installed"""
+
+import sys
+from typing import cast
+
+from pygments import highlight
+from pygments.formatters.terminal import TerminalFormatter
+from pygments.lexers import get_lexer_by_name
+
+
+def colorize(output: str, lexer_name: str) -> str:
+    """Return the output highlighted for terminal if Pygments is available"""
+    if not highlight or not sys.stdout.isatty():
+        return output
+    lexer = get_lexer_by_name(lexer_name)
+    highlighted = highlight(output, lexer, TerminalFormatter())
+    if "\n" not in output:
+        # see https://github.com/pygments/pygments/issues/1107
+        highlighted = highlighted.rstrip("\n")
+    return cast(str, highlighted)
diff --git a/src/darker/highlighting/without_pygments.py b/src/darker/highlighting/without_pygments.py
new file mode 100644
index 000000000..11bfcb7db
--- /dev/null
+++ b/src/darker/highlighting/without_pygments.py
@@ -0,0 +1,6 @@
+"""Linter output highlighting helper to be used when Pygments is not installed"""
+
+
+def colorize(output: str, lexer_name: str) -> str:  # pylint: disable=unused-argument
+    """Return the output unaltered"""
+    return output
diff --git a/src/darker/linting.py b/src/darker/linting.py
index 5eb06e8b2..f1f7bf595 100644
--- a/src/darker/linting.py
+++ b/src/darker/linting.py
@@ -25,7 +25,7 @@
 from typing import Set, Tuple
 
 from darker.git import WORKTREE, EditedLinenumsDiffer, RevisionRange
-from darker.highlighting import DescriptionLexer, LocationLexer, colorize
+from darker.highlighting import colorize
 
 logger = logging.getLogger(__name__)
 
@@ -83,8 +83,6 @@ def run_linter(
     # assert needed for MyPy (see https://stackoverflow.com/q/57350490/15770)
     assert linter_process.stdout is not None
     edited_linenums_differ = EditedLinenumsDiffer(git_root, revrange)
-    location_lexer = LocationLexer()
-    description_lexer = DescriptionLexer()
     prev_path, prev_linenum = None, 0
     for line in linter_process.stdout:
         path_in_repo, linter_error_linenum, location, description = _parse_linter_line(
@@ -99,5 +97,5 @@ def run_linter(
             if path_in_repo != prev_path or linter_error_linenum > prev_linenum + 1:
                 print()
             prev_path, prev_linenum = path_in_repo, linter_error_linenum
-            print(colorize(location, location_lexer), end=" ")
-            print(colorize(description, description_lexer))
+            print(colorize(location, "lint_location"), end=" ")
+            print(colorize(description, "lint_description"))
diff --git a/src/darker/tests/test_highlighting.py b/src/darker/tests/test_highlighting.py
index 617913f05..14d5067bf 100644
--- a/src/darker/tests/test_highlighting.py
+++ b/src/darker/tests/test_highlighting.py
@@ -1,12 +1,46 @@
 """Unit tests for :mod:`darker.highlighting`"""
 
+import sys
 from unittest.mock import Mock, patch
 
 import pytest
 from pygments.token import Token
 
-from darker import highlighting
-from darker.highlighting import Python3Lexer
+from darker.highlighting import lexers, with_pygments, without_pygments
+
+
+def test_colorize_import_without_pygments():
+    """Dummy ``colorize()`` is used if Pygments isn't available"""
+    modules = sys.modules.copy()
+    del modules["darker.highlighting"]
+    # cause an ImportError for `import pygments`:
+    modules["pygments"] = None  # type: ignore[assignment]
+    with patch.dict(sys.modules, modules, clear=True):
+        # pylint: disable=import-outside-toplevel
+
+        from darker.highlighting import colorize
+
+        assert colorize == without_pygments.colorize
+
+
+def test_colorize_import_with_pygments():
+    """The real ``colorize()`` is used if Pygments is available"""
+    assert "pygments" in sys.modules
+    modules = sys.modules.copy()
+    del modules["darker.highlighting"]
+    with patch.dict(sys.modules, modules, clear=True):
+        # pylint: disable=import-outside-toplevel
+
+        from darker.highlighting import colorize
+
+        assert colorize == with_pygments.colorize
+
+
+def test_without_pygments_colorize():
+    """``colorize()`` does nothing when Pygments isn't available"""
+    result = without_pygments.colorize("print(42)", "python")
+
+    assert result == "print(42)"
 
 
 @pytest.mark.parametrize(
@@ -14,20 +48,32 @@
     [
         (
             "except RuntimeError:",
-            Python3Lexer(),
+            "python",
             True,
             "\x1b[34mexcept\x1b[39;49;00m \x1b[36mRuntimeError\x1b[39;49;00m:",
         ),
-        ("except RuntimeError:", Python3Lexer(), False, "except RuntimeError:"),
-        ("a = 1", Python3Lexer(), True, "a = \x1b[34m1\x1b[39;49;00m"),
-        ("a = 1\n", Python3Lexer(), True, "a = \x1b[34m1\x1b[39;49;00m\n"),
+        ("except RuntimeError:", "python", False, "except RuntimeError:"),
+        ("a = 1", "python", True, "a = \x1b[34m1\x1b[39;49;00m"),
+        ("a = 1\n", "python", True, "a = \x1b[34m1\x1b[39;49;00m\n"),
+        (
+            "- a\n+ b\n",
+            "diff",
+            True,
+            "\x1b[91m- a\x1b[39;49;00m\n\x1b[32m+ b\x1b[39;49;00m\n",
+        ),
+        (
+            "- a\n+ b\n",
+            "diff",
+            True,
+            "\x1b[91m- a\x1b[39;49;00m\n\x1b[32m+ b\x1b[39;49;00m\n",
+        ),
     ],
 )
 def test_colorize(text, lexer, tty, expect):
     """``colorize()`` produces correct highlighted terminal output"""
     with patch("sys.stdout.isatty", Mock(return_value=tty)):
 
-        result = highlighting.colorize(text, lexer)
+        result = with_pygments.colorize(text, lexer)
 
     assert result == expect
 
@@ -61,7 +107,7 @@ def test_colorize(text, lexer, tty, expect):
 )
 def test_location_lexer(text, expect):
     """Linter "path:linenum:colnum:" prefixes are lexed correctly"""
-    location_lexer = highlighting.LocationLexer()
+    location_lexer = lexers.LocationLexer()
 
     result = list(location_lexer.get_tokens_unprocessed(text))
 
@@ -217,7 +263,7 @@ def test_location_lexer(text, expect):
 )
 def test_description_lexer(text, expect):
     """The description parts of linter output are lexed correctly"""
-    description_lexer = highlighting.DescriptionLexer()
+    description_lexer = lexers.DescriptionLexer()
 
     result = list(description_lexer.get_tokens_unprocessed(text))