Conditionally remove workaround for Latin-1 Stardoc.

Fixes #818

MODULE.bazel

@@ -40,6 +40,7 @@ bazel_dep(name = "rules_go", version = "0.52.0", dev_dependency = True)
 bazel_dep(name = "abseil-py", version = "2.1.0", dev_dependency = True)
 bazel_dep(name = "gazelle", version = "0.41.0", dev_dependency = True)
 bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True)
+bazel_dep(name = "bazel_features", version = "1.25.0", dev_dependency = True)
 
 # Bring in the local_config_cc repository.  It’s needed for
 # //emacs:windows_cc_toolchain.

docs/generate.py

@@ -36,14 +36,15 @@
 def main() -> None:
     """Main function."""
     parser = argparse.ArgumentParser(allow_abbrev=False)
+    parser.add_argument('--utf8', action='store_true')
     parser.add_argument('input', type=pathlib.Path)
     parser.add_argument('output', type=pathlib.Path)
     opts = parser.parse_args()
     module = stardoc_output_pb2.ModuleInfo.FromString(opts.input.read_bytes())
     # Force Unix-style line endings for consistent results.  See
     # https://github.com/bazelbuild/stardoc/issues/110.
     with opts.output.open(mode='xt', encoding='utf-8', newline='\n') as file:
-        generator = _Generator(file)
+        generator = _Generator(file, utf8=opts.utf8)
         generator.run(module)
 
 
@@ -71,14 +72,15 @@ class _Generator:
         True: 'mandatory',
     }
 
-    def __init__(self, file: io.TextIOBase):
+    def __init__(self, file: io.TextIOBase, *, utf8: bool):
         self._file = file
+        self._utf8 = utf8
 
     def run(self, module: stardoc_output_pb2.ModuleInfo) -> None:
         """Writes the generated Org Mode output."""
         doc = module.module_docstring
         if len(doc) > 100 and '\n' in doc:
-            self._write(_markdown(doc))
+            self._write(self._markdown(doc))
         for rule in module.rule_info:
             self._rule(rule)
         for provider in module.provider_info:
@@ -101,7 +103,7 @@ def _rule(self, rule: stardoc_output_pb2.RuleInfo) -> None:
         self._write(f'#+ATTR_TEXINFO: :options Rule {name} ({attrs})\n')
         self._write('#+BEGIN_deffn\n')
         self._load(rule.origin_key)
-        self._write(_markdown(rule.doc_string).lstrip())
+        self._write(self._markdown(rule.doc_string).lstrip())
         self._write(f'The ~{name}~ rule supports the following attributes:\n\n')
         for attr in rule.attribute:
             self._attribute(attr)
@@ -114,7 +116,7 @@ def _function(self, func: stardoc_output_pb2.StarlarkFunctionInfo) -> None:
         self._write(f'#+ATTR_TEXINFO: :options {name} ({params})\n')
         self._write('#+BEGIN_defun\n')
         self._load(func.origin_key)
-        self._write(_markdown(func.doc_string).lstrip())
+        self._write(self._markdown(func.doc_string).lstrip())
         for param in func.parameter:
             self._parameter(param)
         returns = getattr(func, 'return').doc_string
@@ -125,7 +127,7 @@ def _function(self, func: stardoc_output_pb2.StarlarkFunctionInfo) -> None:
         self._write('#+END_defun\n\n')
 
     def _parameter(self, param: stardoc_output_pb2.FunctionParamInfo) -> None:
-        doc = _markdown(param.doc_string).strip()
+        doc = self._markdown(param.doc_string).strip()
         if not doc.endswith('.'):
             raise ValueError(
                 f'documentation string {doc!r} should end with a period')
@@ -141,10 +143,10 @@ def _provider(self, provider: stardoc_output_pb2.ProviderInfo) -> None:
         self._write(f'#+ATTR_TEXINFO: :options Provider {name} {fields}\n')
         self._write('#+BEGIN_deftp\n')
         self._load(provider.origin_key)
-        self._write(_markdown(provider.doc_string).lstrip())
+        self._write(self._markdown(provider.doc_string).lstrip())
         self._write(f'The ~{name}~ provider has the following fields:\n\n')
         for field in provider.field_info:
-            doc = _markdown(field.doc_string).strip()
+            doc = self._markdown(field.doc_string).strip()
             if not doc.endswith('.'):
                 raise ValueError(
                     f'documentation string {doc!r} should end with a period')
@@ -158,7 +160,7 @@ def _aspect(self, aspect: stardoc_output_pb2.AspectInfo) -> None:
         self._write(f'#+ATTR_TEXINFO: :options Aspect {name} {attrs}\n')
         self._write('#+BEGIN_deffn\n')
         self._load(aspect.origin_key)
-        self._write(_markdown(aspect.doc_string).lstrip())
+        self._write(self._markdown(aspect.doc_string).lstrip())
         if aspect.aspect_attribute:
             attrs = ', '.join(f'~{a}~' for a in aspect.aspect_attribute)
             self._write(f'This aspect propagates along the following '
@@ -177,7 +179,7 @@ def _extension(self, ext: stardoc_output_pb2.ModuleExtensionInfo) -> None:
         self._write(
             f'{name} = use_extension("{ext.origin_key.file}", "{name}")\n')
         self._write('#+END_SRC\n\n')
-        self._write(_markdown(ext.doc_string).lstrip())
+        self._write(self._markdown(ext.doc_string).lstrip())
         self._write(f'The ~{name}~ module extension '
                     f'provides the following tag classes:\n\n')
         for tag in ext.tag_class:
@@ -192,7 +194,7 @@ def _tag_class(self, extension_name: str,
         self._write(f'#+ATTR_TEXINFO: :options '
                     f'{{Tag class}} {extension_name} {name} ({attrs})\n')
         self._write('#+BEGIN_defop\n')
-        self._write(_markdown(tag.doc_string).lstrip())
+        self._write(self._markdown(tag.doc_string).lstrip())
         self._write(
             f'The ~{name}~ tag class supports the following attributes:\n\n')
         for attr in tag.attribute:
@@ -207,7 +209,7 @@ def _repo_rule(self, rule: stardoc_output_pb2.RepositoryRuleInfo) -> None:
             f'#+ATTR_TEXINFO: :options {{Repository rule}} {name} ({attrs})\n')
         self._write('#+BEGIN_deffn\n')
         self._load(rule.origin_key)
-        self._write(_markdown(rule.doc_string).lstrip())
+        self._write(self._markdown(rule.doc_string).lstrip())
         self._write(f'The ~{name}~ repository rule '
                     f'supports the following attributes:\n\n')
         for attr in rule.attribute:
@@ -225,7 +227,7 @@ def _macro(self, macro: stardoc_output_pb2.MacroInfo) -> None:
         self._write(f'#+ATTR_TEXINFO: :options {name} ({attrs})\n')
         self._write('#+BEGIN_defmac\n')
         self._load(macro.origin_key)
-        self._write(_markdown(macro.doc_string).lstrip())
+        self._write(self._markdown(macro.doc_string).lstrip())
         self._write(
             f'The ~{name}~ macro supports the following attributes:\n\n')
         for attr in macro.attribute:
@@ -244,7 +246,7 @@ def _load(self, key: stardoc_output_pb2.OriginKey) -> None:
     def _attribute(self, attr: stardoc_output_pb2.AttributeInfo) -> None:
         if attr.doc_string.startswith('Deprecated;'):
             return
-        doc = _markdown(attr.doc_string).strip()
+        doc = self._markdown(attr.doc_string).strip()
         if not doc.endswith('.'):
             raise ValueError(
                 f'documentation string {doc!r} should end with a period')
@@ -266,16 +268,17 @@ def _item(self, text: str) -> None:
     def _write(self, text: str) -> None:
         self._file.write(text)
 
-
-def _markdown(text: str) -> str:
-    """Convert a Markdown snippet to Org-mode."""
-    # Bazel (including Stardoc) interprets all files as Latin-1,
-    # cf. https://bazel.build/concepts/build-files.  However, our files all use
-    # UTF-8, leading to double encoding.  Reverse that effect here.
-    text = text.strip().encode('latin-1').decode('utf-8')
-    document = commonmark.Parser().parse(text)
-    text = _OrgRenderer().render(document)
-    return text + '\n'
+    def _markdown(self, text: str) -> str:
+        """Convert a Markdown snippet to Org-mode."""
+        text = text.strip()
+        # Bazel before 8.1 (including Stardoc) interprets all files as Latin-1,
+        # cf. https://bazel.build/concepts/build-files.  However, our files all
+        # use UTF-8, leading to double encoding.  Reverse that effect here.
+        if not self._utf8:
+            text = text.encode('latin-1').decode('utf-8')
+        document = commonmark.Parser().parse(text)
+        text = _OrgRenderer().render(document)
+        return text + '\n'
 
 
 def _fill(text: str, *,

docs/merged_manual.bzl

@@ -14,6 +14,7 @@
 
 """Defines the internal `merged_manual` rule."""
 
+load("@bazel_features//:features.bzl", "bazel_features")
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("@bazel_skylib//lib:sets.bzl", "sets")
 
@@ -24,11 +25,15 @@ def _merged_manual_impl(ctx):
     roots = sets.make()
     for bin in ctx.files.includes:
         org = ctx.actions.declare_file(paths.replace_extension(bin.basename, ".org"), sibling = bin)
+        args = ctx.actions.args()
+        if bazel_features.docs.utf8_enabled:
+            args.add("--utf8")
+        args.add("--").add(bin).add(org)
         ctx.actions.run(
             outputs = [org],
             inputs = [bin],
             executable = ctx.executable._generate,
-            arguments = [ctx.actions.args().add("--").add(bin).add(org)],
+            arguments = [args],
             mnemonic = "GenOrg",
             progress_message = "Generating Org file %{output}",
             toolchain = None,