Allow div around dt/dd, including markdown ones. Fixes #2225

bikeshed/h/__init__.py

@@ -41,6 +41,8 @@
     isNormative,
     isOddNode,
     moveContents,
+    nextSiblingElement,
+    nextSiblingNode,
     nodeIter,
     outerHTML,
     parentElement,
@@ -56,6 +58,7 @@
     replaceContents,
     replaceMacros,
     replaceNode,
+    replaceWithContents,
     safeID,
     scopingElements,
     sectionName,

bikeshed/h/dom.py

@@ -208,6 +208,19 @@ def parentElement(el):
     return el.getparent()
 
 
+def nextSiblingNode(el):
+    return el.getnext()
+
+
+def nextSiblingElement(el):
+    while True:
+        next = nextSiblingNode(el)
+        if next is None:
+            return None
+        if isElement(next):
+            return next
+
+
 def appendChild(parent, *children):
     # Appends either text or an element.
     children = list(flatten(children))
@@ -251,7 +264,7 @@ def insertBefore(target, *els):
     parent = target.getparent()
     index = parent.index(target)
     prevSibling = parent[index - 1] if index > 0 else None
-    for el in els:
+    for el in flatten(els):
         if isinstance(el, str):
             if prevSibling is not None:
                 prevSibling.tail = (prevSibling.tail or "") + el
@@ -266,7 +279,7 @@ def insertBefore(target, *els):
 
 def insertAfter(target, *els):
     parent = target.getparent()
-    for el in els:
+    for el in flatten(els):
         if isinstance(el, str):
             target.tail = (target.tail or "") + el
         else:
@@ -311,6 +324,10 @@ def replaceContents(el, newElements):
     return appendContents(el, newElements)
 
 
+def replaceWithContents(el):
+    return replaceNode(el, childNodes(el, clear=True))
+
+
 def moveContents(toEl, fromEl):
     replaceContents(toEl, fromEl)
     fromEl.text = ""

bikeshed/h/serializer.py

@@ -1,4 +1,5 @@
 import io
+import itertools
 
 from . import dom
 
@@ -100,16 +101,18 @@ def unfuckName(self, n):
         return n
 
     def groupIntoBlocks(self, nodes):
-        collect = []
+        nonBlockNodes = []
         for node in nodes:
             if self.isElement(node) and self.isBlockElement(node.tag):
-                yield collect
-                collect = []
+                if nonBlockNodes:
+                    yield nonBlockNodes
+                    nonBlockNodes = []
                 yield node
                 continue
             else:
-                collect.append(node)
-        yield collect
+                nonBlockNodes.append(node)
+        if nonBlockNodes:
+            yield nonBlockNodes
 
     def fixWS(self, text):
         import string
@@ -164,6 +167,16 @@ def isInlineElement(self, tag):
     def isBlockElement(self, tag):
         return not self.isInlineElement(tag)
 
+    def needsEndTag(self, el, nextEl=None):
+        if el.tag not in self.omitEndTagEls:
+            return True
+        if el.tag in ["dt", "dd"]:
+            if nextEl is None:
+                return False
+            if self.isElement(nextEl) and nextEl.tag in ["dt", "dd"]:
+                return False
+            return True
+
     def justWS(self, block):
         if self.isElement(block):
             return False
@@ -216,7 +229,7 @@ def _categorizeBlockChildren(self, el):
                 return "blocks", self._blocksFromChildren(children)
         return "inlines", children
 
-    def _writeBlockElement(self, tag, el, write, indent):
+    def _writeBlockElement(self, tag, el, write, indent, nextEl):
         # Dropping pure-WS anonymous blocks.
         # This maintains whitespace between *inline* elements, which is required.
         # It just avoids serializing a line of "inline content" that's just WS.
@@ -226,34 +239,34 @@ def _writeBlockElement(self, tag, el, write, indent):
             # Empty of text and children
             write(" " * indent)
             self.startTag(tag, el, write)
-            if el.tag not in self.omitEndTagEls:
+            if self.needsEndTag(el, nextEl):
                 self.endTag(tag, write)
         elif contentsType == "inlines":
             # Contains only inlines, print accordingly
             write(" " * indent)
             self.startTag(tag, el, write)
             self._serializeEl(contents, write, inline=True)
-            if el.tag not in self.omitEndTagEls:
+            if self.needsEndTag(el, nextEl):
                 self.endTag(tag, write)
             return
         else:
             # Otherwise I'm a block that contains at least one block
             write(" " * indent)
             self.startTag(tag, el, write)
-            for block in contents:
+            for block, nextBlock in pairwise(contents):
                 if isinstance(block, list):
                     # is an array of inlines
                     if len(block) > 0:
                         write("\n" + (" " * (indent + 1)))
                         self._serializeEl(block, write, inline=True)
                 else:
                     write("\n")
-                    self._serializeEl(block, write, indent=indent + 1)
-            if tag not in self.omitEndTagEls:
+                    self._serializeEl(block, write, indent=indent + 1, nextEl=nextBlock)
+            if self.needsEndTag(el, nextEl):
                 write("\n" + (" " * indent))
                 self.endTag(tag, write)
 
-    def _serializeEl(self, el, write, indent=0, pre=False, inline=False):
+    def _serializeEl(self, el, write, indent=0, pre=False, inline=False, nextEl=None):
         if isinstance(el, list):
             tag = "[]"
         else:
@@ -268,4 +281,11 @@ def _serializeEl(self, el, write, indent=0, pre=False, inline=False):
         elif inline or self.isInlineElement(el):
             self._writeInlineElement(tag, el, write, inline)
         else:
-            self._writeBlockElement(tag, el, write, indent)
+            self._writeBlockElement(tag, el, write, indent, nextEl)
+
+
+def pairwise(iterable):
+    # pairwise('ABCDEFG') --> AB BC CD DE EF FG GNone
+    a, b = itertools.tee(iterable)
+    next(b, None)
+    return itertools.zip_longest(a, b)

bikeshed/unsortedJunk.py

@@ -11,6 +11,7 @@
     E,
     addClass,
     appendChild,
+    childElements,
     childNodes,
     clearContents,
     closestAncestor,
@@ -40,6 +41,7 @@
     relevantHeadings,
     removeAttr,
     replaceContents,
+    replaceWithContents,
     replaceNode,
     safeID,
     textContent,
@@ -1166,20 +1168,49 @@ def cleanupHTML(doc):
 
         # Allow MD-generated lists to be surrounded by HTML list containers,
         # so you can add classes/etc without an extraneous wrapper.
-        if el.tag in ["ol", "ul", "dl"]:
-            onlyChild = hasOnlyChild(el)
-            if (
-                onlyChild is not None
-                and el.tag == onlyChild.tag
-                and el.get("data-md") is None
-                and onlyChild.get("data-md") is not None
-            ):
-                # The md-generated list container is featureless,
-                # so we can just throw it away and move its children into its parent.
-                nestedLists.append(onlyChild)
+        if el.tag in ["ol", "ul", "dl"] and el.get("data-md") is None:
+            if el.tag in ["ol", "ul"]:
+                onlyChild = hasOnlyChild(el)
+                if (
+                    onlyChild is not None
+                    and el.tag == onlyChild.tag
+                    and onlyChild.get("data-md") is not None
+                ):
+                    # The md-generated list container is featureless,
+                    # so we can just throw it away and move its children into its parent.
+                    nestedLists.append(onlyChild)
             else:
-                # Remove any lingering data-md attributes on lists that weren't using this container replacement thing.
-                removeAttr(el, "data-md")
+                # dls can contain both dt/dds
+                # (which'll make an md-generated dl)
+                # and divs, so I need to account for multiple children
+                for child in childElements(el):
+                    if child.tag == "dl" and child.get("data-md") is not None:
+                        nestedLists.append(child)
+                    elif child.tag == "div":
+                        pass
+                    elif child.tag in ["dt", "dd"]:
+                        pass
+                    else:
+                        # misnested element; leave alone for now
+                        pass
+
+        # HTML allows dt/dd to be grouped by a div, so recognize
+        # when a markdown-generated dl has a div parent and dl grandparent
+        # and remove it.
+        if (
+            el.tag == "dl"
+            and el.get("data-md") is not None
+            and parentElement(el).tag == "div"
+            and parentElement(parentElement(el)).tag == "dl"
+            ):
+            # Also featureless and can be safely thrown away
+            # with its children merged into the parent div
+            nestedLists.append(el)
+
+
+        # Remove any lingering data-md attributes on lists
+        if el.tag in ["ol", "ul", "dl"] and el.get("data-md") is not None:
+            removeAttr(el, "data-md")
 
         # Mark pre.idl blocks as .def, for styling
         if el.tag == "pre" and hasClass(el, "idl") and not hasClass(el, "def"):
@@ -1311,10 +1342,7 @@ def cleanupHTML(doc):
         parent = parentElement(el)
         prependChild(parent, el)
     for el in nestedLists:
-        children = childNodes(el, clear=True)
-        parent = parentElement(el)
-        clearContents(parent)
-        appendChild(parent, *children)
+        replaceWithContents(el)
     for el in flattenEls:
         moveContents(fromEl=el[0], toEl=el)
 

tests/github/w3c/ServiceWorker/publish/service_worker/WD-service-workers-20160830/index.html

@@ -3162,7 +3162,7 @@ <h3 class="heading settled" id="soft-update-algorithm"><span class="content">Sof
       <dl>
        <dt>Input
        <dd><var>registration</var>, a <a href="#dfn-service-worker-registration" id="ref-for-dfn-service-worker-registration⑤①">service worker registration</a>
-       <dd><em>force bypass cache flag</em>, an optional flag unset by default
+       <dd><em>force bypass cache flag</em>, an optional flag unset by default</dd>
        <p class="note" role="note">Implementers may use the <em>force bypass cache flag</em> to aid debugging (e.g. invocations from developer tools), and other specifications that extend service workers may also use the flag on their own needs.</p>
        <dt>Output
        <dd>None
@@ -3327,7 +3327,7 @@ <h3 class="heading settled" id="run-service-worker-algorithm"><span class="conte
          <dt>The <a data-link-type="dfn" href="https://html.spec.whatwg.org/multipage/webappapis.html#responsible-event-loop" id="ref-for-responsible-event-loop⑨">responsible event loop</a>
          <dd>Return <var>workerEventLoop</var>.
          <dt>The <a data-link-type="dfn">referrer source</a>
-         <dd>Return <var>serviceWorker</var>’s <a href="#dfn-script-url" id="ref-for-dfn-script-url⑦">script url</a>.
+         <dd>Return <var>serviceWorker</var>’s <a href="#dfn-script-url" id="ref-for-dfn-script-url⑦">script url</a>.</dd>
          <p class="issue" id="issue-b739bfcf"><a class="self-link" href="#issue-b739bfcf"></a>Remove this definition after sorting out the referencing sites.</p>
          <dt>The <a data-link-type="dfn" href="https://html.spec.whatwg.org/multipage/webappapis.html#api-url-character-encoding" id="ref-for-api-url-character-encoding">API URL character encoding</a>
          <dd>Return UTF-8.

tests/github/w3c/ServiceWorker/publish/service_worker_1/WD-service-workers-1-20161011/index.html

@@ -2857,7 +2857,7 @@ <h3 class="heading settled" id="soft-update-algorithm"><span class="content">Sof
       <dl>
        <dt>Input
        <dd><var>registration</var>, a <a href="#dfn-service-worker-registration" id="ref-for-dfn-service-worker-registration⑤⓪">service worker registration</a>
-       <dd><em>force bypass cache flag</em>, an optional flag unset by default
+       <dd><em>force bypass cache flag</em>, an optional flag unset by default</dd>
        <p class="note" role="note">Implementers may use the <em>force bypass cache flag</em> to aid debugging (e.g. invocations from developer tools), and other specifications that extend service workers may also use the flag on their own needs.</p>
        <dt>Output
        <dd>None
@@ -3022,7 +3022,7 @@ <h3 class="heading settled" id="run-service-worker-algorithm"><span class="conte
          <dt>The <a data-link-type="dfn" href="https://html.spec.whatwg.org/multipage/webappapis.html#responsible-event-loop" id="ref-for-responsible-event-loop⑨">responsible event loop</a>
          <dd>Return <var>workerEventLoop</var>.
          <dt>The <a data-link-type="dfn">referrer source</a>
-         <dd>Return <var>serviceWorker</var>’s <a href="#dfn-script-url" id="ref-for-dfn-script-url⑦">script url</a>.
+         <dd>Return <var>serviceWorker</var>’s <a href="#dfn-script-url" id="ref-for-dfn-script-url⑦">script url</a>.</dd>
          <p class="issue" id="issue-b739bfcf"><a class="self-link" href="#issue-b739bfcf"></a>Remove this definition after sorting out the referencing sites.</p>
          <dt>The <a data-link-type="dfn" href="https://html.spec.whatwg.org/multipage/webappapis.html#api-url-character-encoding" id="ref-for-api-url-character-encoding">API URL character encoding</a>
          <dd>Return UTF-8.

tests/github/w3c/clipboard-apis/index.bs

@@ -945,17 +945,13 @@ urlPrefix: https://w3c.github.io/FileAPI/#dfn-; type: dfn;
 	These clipboard permissions are <a>powerful feature</a>s
 	permission-related algorithms and types are defined as follows:
 
-	<dl>
-	<dt>
-		<a>permission descriptor type</a>
-	</dt>
-	<dd>
+	: <a>permission descriptor type</a>
+	::
 		<pre class="idl">
 			dictionary ClipboardPermissionDescriptor : PermissionDescriptor {
 				boolean allowWithoutGesture = false;
 			};
 		</pre>
-	</dd>
 
 	There are 4 clipboard permissions: