nixos-render-docs: add options asciidoc converter

same reasoning as for the earlier commonmark converter.

nixos/lib/make-options-doc/default.nix

@@ -91,11 +91,14 @@ let
 in rec {
   inherit optionsNix;
 
-  optionsAsciiDoc = pkgs.runCommand "options.adoc" {} ''
-    ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \
-      --format asciidoc \
+  optionsAsciiDoc = pkgs.runCommand "options.adoc" {
+    nativeBuildInputs = [ pkgs.nixos-render-docs ];
+  } ''
+    nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \
+      --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \
+      --revision ${lib.escapeShellArg revision} \
       ${optionsJSON}/share/doc/nixos/options.json \
-      > $out
+      $out
   '';
 
   optionsCommonMark = pkgs.runCommand "options.md" {

nixos/lib/make-options-doc/generateDoc.py

@@ -1,83 +0,0 @@
-import argparse
-import json
-import sys
-
-formats = ['asciidoc']
-
-parser = argparse.ArgumentParser(
-    description = 'Generate documentation for a set of JSON-formatted NixOS options'
-)
-parser.add_argument(
-    'nix_options_path',
-    help = 'a path to a JSON file containing the NixOS options'
-)
-parser.add_argument(
-    '-f',
-    '--format',
-    choices = formats,
-    required = True,
-    help = f'the documentation format to generate'
-)
-
-args = parser.parse_args()
-
-class OptionsEncoder(json.JSONEncoder):
-    def encode(self, obj):
-        # Unpack literal expressions and other Nix types.
-        # Don't escape the strings: they were escaped when initially serialized to JSON.
-        if isinstance(obj, dict):
-            _type = obj.get('_type')
-            if _type is not None:
-                if _type == 'literalExpression' or _type == 'literalDocBook':
-                    return obj['text']
-
-                if _type == 'derivation':
-                    return obj['name']
-
-                raise Exception(f'Unexpected type `{_type}` in {json.dumps(obj)}')
-
-        return super().encode(obj)
-
-# TODO: declarations: link to github
-def generate_asciidoc(options):
-    for (name, value) in options.items():
-        print(f'== {name}')
-        print()
-        print(value['description'])
-        print()
-        print('[discrete]')
-        print('=== details')
-        print()
-        print(f'Type:: {value["type"]}')
-        if 'default' in value:
-            print('Default::')
-            print('+')
-            print('----')
-            print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
-            print('----')
-            print()
-        else:
-            print('No Default:: {blank}')
-        if value['readOnly']:
-            print('Read Only:: {blank}')
-        else:
-            print()
-        if 'example' in value:
-            print('Example::')
-            print('+')
-            print('----')
-            print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
-            print('----')
-            print()
-        else:
-            print('No Example:: {blank}')
-        print()
-
-with open(args.nix_options_path) as nix_options_json:
-    options = json.load(nix_options_json)
-
-    if args.format == 'asciidoc':
-        generate_asciidoc(options)
-    else:
-        raise Exception(f'Unsupported documentation format `--format {args.format}`')
-

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py

@@ -0,0 +1,262 @@
+from collections.abc import Mapping, MutableMapping, Sequence
+from dataclasses import dataclass
+from typing import Any, cast, Optional
+from urllib.parse import quote
+
+from .md import Renderer
+
+import markdown_it
+from markdown_it.token import Token
+from markdown_it.utils import OptionsDict
+
+_asciidoc_escapes = {
+    # escape all dots, just in case one is pasted at SOL
+    ord('.'): "{zwsp}.",
+    # may be replaced by typographic variants
+    ord("'"): "{apos}",
+    ord('"'): "{quot}",
+    # passthrough character
+    ord('+'): "{plus}",
+    # table marker
+    ord('|'): "{vbar}",
+    # xml entity reference
+    ord('&'): "{amp}",
+    # crossrefs. < needs extra escaping because links break in odd ways if they start with it
+    ord('<'): "{zwsp}+<+{zwsp}",
+    ord('>'): "{gt}",
+    # anchors, links, block attributes
+    ord('['): "{startsb}",
+    ord(']'): "{endsb}",
+    # superscript, subscript
+    ord('^'): "{caret}",
+    ord('~'): "{tilde}",
+    # bold
+    ord('*'): "{asterisk}",
+    # backslash
+    ord('\\'): "{backslash}",
+    # inline code
+    ord('`'): "{backtick}",
+}
+def asciidoc_escape(s: str) -> str:
+    s = s.translate(_asciidoc_escapes)
+    # :: is deflist item, ;; is has a replacement but no idea why
+    return s.replace("::", "{two-colons}").replace(";;", "{two-semicolons}")
+
+@dataclass(kw_only=True)
+class List:
+    head: str
+
+@dataclass()
+class Par:
+    sep: str
+    block_delim: str
+    continuing: bool = False
+
+class AsciiDocRenderer(Renderer):
+    __output__ = "asciidoc"
+
+    _parstack: list[Par]
+    _list_stack: list[List]
+    _attrspans: list[str]
+
+    def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
+        super().__init__(manpage_urls, parser)
+        self._parstack = [ Par("\n\n", "====") ]
+        self._list_stack = []
+        self._attrspans = []
+
+    def _enter_block(self, is_list: bool) -> None:
+        self._parstack.append(Par("\n+\n" if is_list else "\n\n", self._parstack[-1].block_delim + "="))
+    def _leave_block(self) -> None:
+        self._parstack.pop()
+    def _break(self, force: bool = False) -> str:
+        result = self._parstack[-1].sep if force or self._parstack[-1].continuing else ""
+        self._parstack[-1].continuing = True
+        return result
+
+    def _admonition_open(self, kind: str) -> str:
+        pbreak = self._break()
+        self._enter_block(False)
+        return f"{pbreak}[{kind}]\n{self._parstack[-2].block_delim}\n"
+    def _admonition_close(self) -> str:
+        self._leave_block()
+        return f"\n{self._parstack[-1].block_delim}\n"
+
+    def _list_open(self, token: Token, head: str) -> str:
+        attrs = []
+        if (idx := token.attrs.get('start')) is not None:
+            attrs.append(f"start={idx}")
+        if token.meta['compact']:
+            attrs.append('options="compact"')
+        if self._list_stack:
+            head *= len(self._list_stack[0].head) + 1
+        self._list_stack.append(List(head=head))
+        return f"{self._break()}[{','.join(attrs)}]"
+    def _list_close(self) -> str:
+        self._list_stack.pop()
+        return ""
+
+    def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+             env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        return asciidoc_escape(token.content)
+    def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                       env: MutableMapping[str, Any]) -> str:
+        return self._break()
+    def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                        env: MutableMapping[str, Any]) -> str:
+        return ""
+    def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        return " +\n"
+    def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        return f" "
+    def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                    env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        return f"``{asciidoc_escape(token.content)}``"
+    def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                   env: MutableMapping[str, Any]) -> str:
+        return self.fence(token, tokens, i, options, env)
+    def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}["
+    def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                   env: MutableMapping[str, Any]) -> str:
+        return "]"
+    def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                       env: MutableMapping[str, Any]) -> str:
+        self._enter_block(True)
+        # allow the next token to be a block or an inline.
+        return f'\n{self._list_stack[-1].head} {{empty}}'
+    def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                        env: MutableMapping[str, Any]) -> str:
+        self._leave_block()
+        return "\n"
+    def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                         env: MutableMapping[str, Any]) -> str:
+        return self._list_open(token, '*')
+    def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                          env: MutableMapping[str, Any]) -> str:
+        return self._list_close()
+    def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                env: MutableMapping[str, Any]) -> str:
+        return "__"
+    def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                 env: MutableMapping[str, Any]) -> str:
+        return "__"
+    def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                    env: MutableMapping[str, Any]) -> str:
+        return "**"
+    def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                     env: MutableMapping[str, Any]) -> str:
+        return "**"
+    def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+              env: MutableMapping[str, Any]) -> str:
+        attrs = f"[source,{token.info}]\n" if token.info else ""
+        code = token.content
+        if code.endswith('\n'):
+            code = code[:-1]
+        return f"{self._break(True)}{attrs}----\n{code}\n----"
+    def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                        env: MutableMapping[str, Any]) -> str:
+        pbreak = self._break(True)
+        self._enter_block(False)
+        return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n"
+    def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                         env: MutableMapping[str, Any]) -> str:
+        self._leave_block()
+        return f"\n{self._parstack[-1].block_delim}"
+    def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        return self._admonition_open("NOTE")
+    def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                   env: MutableMapping[str, Any]) -> str:
+        return self._admonition_close()
+    def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                     env: MutableMapping[str, Any]) -> str:
+        return self._admonition_open("CAUTION")
+    def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                      env: MutableMapping[str, Any]) -> str:
+        return self._admonition_close()
+    def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                       env: MutableMapping[str, Any]) -> str:
+        return self._admonition_open("IMPORTANT")
+    def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                        env: MutableMapping[str, Any]) -> str:
+        return self._admonition_close()
+    def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                 env: MutableMapping[str, Any]) -> str:
+        return self._admonition_open("TIP")
+    def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        return self._admonition_close()
+    def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                     env: MutableMapping[str, Any]) -> str:
+        return self._admonition_open("WARNING")
+    def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                      env: MutableMapping[str, Any]) -> str:
+        return self._admonition_close()
+    def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                env: MutableMapping[str, Any]) -> str:
+        return f"{self._break()}[]"
+    def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                 env: MutableMapping[str, Any]) -> str:
+        return ""
+    def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                env: MutableMapping[str, Any]) -> str:
+        return self._break()
+    def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                 env: MutableMapping[str, Any]) -> str:
+        self._enter_block(True)
+        return ":: {empty}"
+    def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                env: MutableMapping[str, Any]) -> str:
+        return ""
+    def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                 env: MutableMapping[str, Any]) -> str:
+        self._leave_block()
+        return "\n"
+    def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                  env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        content = asciidoc_escape(token.content)
+        if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)):
+            return f"link:{quote(url, safe='/:')}[{content}]"
+        return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``"
+    def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                      env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        return f"[[{token.attrs['id']}]]"
+    def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                        env: MutableMapping[str, Any]) -> str:
+        self._parstack[-1].continuing = True
+        (id_part, class_part) = ("", "")
+        if id := token.attrs.get('id'):
+            id_part = f"[[{id}]]"
+        if s := token.attrs.get('class'):
+            if s == 'keycap':
+                class_part = "kbd:["
+                self._attrspans.append("]")
+            else:
+                return super().attr_span_begin(token, tokens, i, options, env)
+        else:
+            self._attrspans.append("")
+        return id_part + class_part
+    def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                      env: MutableMapping[str, Any]) -> str:
+        return self._attrspans.pop()
+    def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                     env: MutableMapping[str, Any]) -> str:
+        return token.markup.replace("#", "=") + " "
+    def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                      env: MutableMapping[str, Any]) -> str:
+        return "\n"
+    def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                          env: MutableMapping[str, Any]) -> str:
+        return self._list_open(token, '.')
+    def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
+                           env: MutableMapping[str, Any]) -> str:
+        return self._list_close()

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py

@@ -8,11 +8,13 @@ from collections.abc import Mapping, MutableMapping, Sequence
 from markdown_it.utils import OptionsDict
 from markdown_it.token import Token
 from typing import Any, Optional
+from urllib.parse import quote
 from xml.sax.saxutils import escape, quoteattr
 
 import markdown_it
 
 from . import parallel
+from .asciidoc import AsciiDocRenderer, asciidoc_escape
 from .commonmark import CommonMarkRenderer
 from .docbook import DocBookRenderer, make_xml_id
 from .manpage import ManpageRenderer, man_escape
@@ -476,6 +478,59 @@ class CommonMarkConverter(BaseConverter):
 
         return "\n".join(result)
 
+class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer):
+    pass
+
+class AsciiDocConverter(BaseConverter):
+    __renderer__ = AsciiDocRenderer
+    __option_block_separator__ = ""
+
+    def _parallel_render_prepare(self) -> Any:
+        return (self._manpage_urls, self._revision, self._markdown_by_default)
+    @classmethod
+    def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter:
+        return cls(*a)
+
+    def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
+        # NOTE this duplicates the old direct-paste behavior, even if it is somewhat
+        # incorrect, since users rely on it.
+        if lit := option_is(option, key, 'literalDocBook'):
+            return [ f"*{key.capitalize()}:* {lit['text']}" ]
+        else:
+            return super()._render_code(option, key)
+
+    def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
+        # NOTE this duplicates the old direct-paste behavior, even if it is somewhat
+        # incorrect, since users rely on it.
+        if isinstance(desc, str) and not self._markdown_by_default:
+            return [ desc ]
+        else:
+            return super()._render_description(desc)
+
+    def _related_packages_header(self) -> list[str]:
+        return [ "__Related packages:__" ]
+
+    def _decl_def_header(self, header: str) -> list[str]:
+        return [ f"__{header}:__\n" ]
+
+    def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:
+        if href is not None:
+            return [ f"* link:{quote(href, safe='/:')}[{asciidoc_escape(name)}]" ]
+        return [ f"* {asciidoc_escape(name)}" ]
+
+    def _decl_def_footer(self) -> list[str]:
+        return []
+
+    def finalize(self) -> str:
+        result = []
+
+        for (name, opt) in self._sorted_options():
+            result.append(f"== {asciidoc_escape(name)}\n")
+            result += opt.lines
+            result.append("\n\n")
+
+        return "\n".join(result)
+
 def _build_cli_db(p: argparse.ArgumentParser) -> None:
     p.add_argument('--manpage-urls', required=True)
     p.add_argument('--revision', required=True)
@@ -498,6 +553,13 @@ def _build_cli_commonmark(p: argparse.ArgumentParser) -> None:
     p.add_argument("infile")
     p.add_argument("outfile")
 
+def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None:
+    p.add_argument('--manpage-urls', required=True)
+    p.add_argument('--revision', required=True)
+    p.add_argument('--markdown-by-default', default=False, action='store_true')
+    p.add_argument("infile")
+    p.add_argument("outfile")
+
 def _run_cli_db(args: argparse.Namespace) -> None:
     with open(args.manpage_urls, 'r') as manpage_urls:
         md = DocBookConverter(
@@ -537,11 +599,24 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None:
         with open(args.outfile, 'w') as f:
             f.write(md.finalize())
 
+def _run_cli_asciidoc(args: argparse.Namespace) -> None:
+    with open(args.manpage_urls, 'r') as manpage_urls:
+        md = AsciiDocConverter(
+            json.load(manpage_urls),
+            revision = args.revision,
+            markdown_by_default = args.markdown_by_default)
+
+        with open(args.infile, 'r') as f:
+            md.add_options(json.load(f))
+        with open(args.outfile, 'w') as f:
+            f.write(md.finalize())
+
 def build_cli(p: argparse.ArgumentParser) -> None:
     formats = p.add_subparsers(dest='format', required=True)
     _build_cli_db(formats.add_parser('docbook'))
     _build_cli_manpage(formats.add_parser('manpage'))
     _build_cli_commonmark(formats.add_parser('commonmark'))
+    _build_cli_asciidoc(formats.add_parser('asciidoc'))
 
 def run_cli(args: argparse.Namespace) -> None:
     if args.format == 'docbook':
@@ -550,5 +625,7 @@ def run_cli(args: argparse.Namespace) -> None:
         _run_cli_manpage(args)
     elif args.format == 'commonmark':
         _run_cli_commonmark(args)
+    elif args.format == 'asciidoc':
+        _run_cli_asciidoc(args)
     else:
         raise RuntimeError('format not hooked up', args)

pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py

@@ -0,0 +1,143 @@
+import nixos_render_docs
+
+from sample_md import sample1
+
+class Converter(nixos_render_docs.md.Converter):
+    __renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer
+
+def test_lists() -> None:
+    c = Converter({})
+    # attaching to the nth ancestor list requires n newlines before the +
+    assert c._render("""\
+- a
+
+  b
+- c
+  - d
+    - e
+
+      1
+
+  f
+""") == """\
+[]
+* {empty}a
++
+b
+
+* {empty}c
++
+[options="compact"]
+** {empty}d
++
+[]
+** {empty}e
++
+1
+
+
++
+f
+"""
+
+def test_full() -> None:
+    c = Converter({ 'man(1)': 'http://example.org' })
+    assert c._render(sample1) == """\
+[WARNING]
+====
+foo
+
+[NOTE]
+=====
+nested
+=====
+
+====
+
+
+link:link[ multiline ]
+
+link:http://example.org[man(1)] reference
+
+[[b]]some [[a]]nested anchors
+
+__emph__ **strong** __nesting emph **and strong** and ``code``__
+
+[]
+* {empty}wide bullet
+
+* {empty}list
+
+
+[]
+. {empty}wide ordered
+
+. {empty}list
+
+
+[options="compact"]
+* {empty}narrow bullet
+
+* {empty}list
+
+
+[options="compact"]
+. {empty}narrow ordered
+
+. {empty}list
+
+
+[quote]
+====
+quotes
+
+[quote]
+=====
+with __nesting__
+
+----
+nested code block
+----
+=====
+
+[options="compact"]
+* {empty}and lists
+
+* {empty}
++
+----
+containing code
+----
+
+
+and more quote
+====
+
+[start=100,options="compact"]
+. {empty}list starting at 100
+
+. {empty}goes on
+
+
+[]
+
+deflist:: {empty}
++
+[quote]
+=====
+with a quote and stuff
+=====
++
+----
+code block
+----
++
+----
+fenced block
+----
++
+text
+
+
+more stuff in same deflist:: {empty}foo
+"""