import collections import json import sys from typing import Any, Dict, List # for MD conversion import mistune import re from xml.sax.saxutils import escape, quoteattr JSON = Dict[str, Any] class Key: def __init__(self, path: List[str]): self.path = path def __hash__(self): result = 0 for id in self.path: result ^= hash(id) return result def __eq__(self, other): return type(self) is type(other) and self.path == other.path Option = collections.namedtuple('Option', ['name', 'value']) # pivot a dict of options keyed by their display name to a dict keyed by their path def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]: result: Dict[Key, Option] = dict() for (name, opt) in options.items(): result[Key(opt['loc'])] = Option(name, opt) return result # pivot back to indexed-by-full-name # like the docbook build we'll just fail if multiple options with differing locs # render to the same option name. def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]: result: Dict[str, Dict] = dict() for (key, opt) in options.items(): if opt.name in result: raise RuntimeError( 'multiple options with colliding ids found', opt.name, result[opt.name]['loc'], opt.value['loc'], ) result[opt.name] = opt.value return result admonitions = { '.warning': 'warning', '.important': 'important', '.note': 'note' } class Renderer(mistune.renderers.BaseRenderer): def _get_method(self, name): try: return super(Renderer, self)._get_method(name) except AttributeError: def not_supported(*args, **kwargs): raise NotImplementedError("md node not supported yet", name, args, **kwargs) return not_supported def text(self, text): return escape(text) def paragraph(self, text): return text + "\n\n" def newline(self): return "\n" def codespan(self, text): return f"{escape(text)}" def block_code(self, text, info=None): info = f" language={quoteattr(info)}" if info is not None else "" return f"\n{escape(text)}" def link(self, link, text=None, title=None): tag = "link" if link[0:1] == '#': if text == "": tag = "xref" attr = "linkend" link = quoteattr(link[1:]) else: # try to faithfully reproduce links that were of the form # in docbook format if text == link: text = "" attr = "xlink:href" link = quoteattr(link) return f"<{tag} {attr}={link}>{text}" def list(self, text, ordered, level, start=None): if ordered: raise NotImplementedError("ordered lists not supported yet") return f"\n{text}\n" def list_item(self, text, level): return f"{text}\n" def block_text(self, text): return text def emphasis(self, text): return f"{text}" def strong(self, text): return f"{text}" def admonition(self, text, kind): if kind not in admonitions: raise NotImplementedError(f"admonition {kind} not supported yet") tag = admonitions[kind] # we don't keep whitespace here because usually we'll contain only # a single paragraph and the original docbook string is no longer # available to restore the trailer. return f"<{tag}>{text.rstrip()}" def block_quote(self, text): return f"
{text}
" def command(self, text): return f"{escape(text)}" def option(self, text): return f"" def file(self, text): return f"{escape(text)}" def var(self, text): return f"{escape(text)}" def env(self, text): return f"{escape(text)}" def manpage(self, page, section): title = f"{escape(page)}" vol = f"{escape(section)}" return f"{title}{vol}" def finalize(self, data): return "".join(data) def p_command(md): COMMAND_PATTERN = r'\{command\}`(.*?)`' def parse(self, m, state): return ('command', m.group(1)) md.inline.register_rule('command', COMMAND_PATTERN, parse) md.inline.rules.append('command') def p_file(md): FILE_PATTERN = r'\{file\}`(.*?)`' def parse(self, m, state): return ('file', m.group(1)) md.inline.register_rule('file', FILE_PATTERN, parse) md.inline.rules.append('file') def p_var(md): VAR_PATTERN = r'\{var\}`(.*?)`' def parse(self, m, state): return ('var', m.group(1)) md.inline.register_rule('var', VAR_PATTERN, parse) md.inline.rules.append('var') def p_env(md): ENV_PATTERN = r'\{env\}`(.*?)`' def parse(self, m, state): return ('env', m.group(1)) md.inline.register_rule('env', ENV_PATTERN, parse) md.inline.rules.append('env') def p_option(md): OPTION_PATTERN = r'\{option\}`(.*?)`' def parse(self, m, state): return ('option', m.group(1)) md.inline.register_rule('option', OPTION_PATTERN, parse) md.inline.rules.append('option') def p_manpage(md): MANPAGE_PATTERN = r'\{manpage\}`(.*?)\((.+?)\)`' def parse(self, m, state): return ('manpage', m.group(1), m.group(2)) md.inline.register_rule('manpage', MANPAGE_PATTERN, parse) md.inline.rules.append('manpage') def p_admonition(md): ADMONITION_PATTERN = re.compile(r'^::: \{([^\n]*?)\}\n(.*?)^:::$\n*', flags=re.MULTILINE|re.DOTALL) def parse(self, m, state): return { 'type': 'admonition', 'children': self.parse(m.group(2), state), 'params': [ m.group(1) ], } md.block.register_rule('admonition', ADMONITION_PATTERN, parse) md.block.rules.append('admonition') md = mistune.create_markdown(renderer=Renderer(), plugins=[ p_command, p_file, p_var, p_env, p_option, p_manpage, p_admonition ]) # converts in-place! def convertMD(options: Dict[str, Any]) -> str: def convertString(path: str, text: str) -> str: try: rendered = md(text) # keep trailing spaces so we can diff the generated XML to check for conversion bugs. return rendered.rstrip() + text[len(text.rstrip()):] except: print(f"error in {path}") raise def optionIs(option: Dict[str, Any], key: str, typ: str) -> bool: if key not in option: return False if type(option[key]) != dict: return False if '_type' not in option[key]: return False return option[key]['_type'] == typ for (name, option) in options.items(): try: if optionIs(option, 'description', 'mdDoc'): option['description'] = convertString(name, option['description']['text']) elif markdownByDefault: option['description'] = convertString(name, option['description']) if optionIs(option, 'example', 'literalMD'): docbook = convertString(name, option['example']['text']) option['example'] = { '_type': 'literalDocBook', 'text': docbook } if optionIs(option, 'default', 'literalMD'): docbook = convertString(name, option['default']['text']) option['default'] = { '_type': 'literalDocBook', 'text': docbook } except Exception as e: raise Exception(f"Failed to render option {name}: {str(e)}") return options warningsAreErrors = False errorOnDocbook = False markdownByDefault = False optOffset = 0 for arg in sys.argv[1:]: if arg == "--warnings-are-errors": optOffset += 1 warningsAreErrors = True if arg == "--error-on-docbook": optOffset += 1 errorOnDocbook = True if arg == "--markdown-by-default": optOffset += 1 markdownByDefault = True options = pivot(json.load(open(sys.argv[1 + optOffset], 'r'))) overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r'))) # fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir for (k, v) in options.items(): # The _module options are not declared in nixos/modules if v.value['loc'][0] != "_module": v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}' if isinstance(s, str) else s, v.value['declarations'])) # merge both descriptions for (k, v) in overrides.items(): cur = options.setdefault(k, v).value for (ok, ov) in v.value.items(): if ok == 'declarations': decls = cur[ok] for d in ov: if d not in decls: decls += [d] elif ok == "type": # ignore types of placeholder options if ov != "_unspecified" or cur[ok] == "_unspecified": cur[ok] = ov elif ov is not None or cur.get(ok, None) is None: cur[ok] = ov severity = "error" if warningsAreErrors else "warning" def is_docbook(o, key): val = o.get(key, {}) if not isinstance(val, dict): return False return val.get('_type', '') == 'literalDocBook' # check that every option has a description hasWarnings = False hasErrors = False hasDocBookErrors = False for (k, v) in options.items(): if errorOnDocbook: if isinstance(v.value.get('description', {}), str): hasErrors = True hasDocBookErrors = True print( f"\x1b[1;31merror: option {v.name} description uses DocBook\x1b[0m", file=sys.stderr) elif is_docbook(v.value, 'defaultText'): hasErrors = True hasDocBookErrors = True print( f"\x1b[1;31merror: option {v.name} default uses DocBook\x1b[0m", file=sys.stderr) elif is_docbook(v.value, 'example'): hasErrors = True hasDocBookErrors = True print( f"\x1b[1;31merror: option {v.name} example uses DocBook\x1b[0m", file=sys.stderr) if v.value.get('description', None) is None: hasWarnings = True print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr) v.value['description'] = "This option has no description." if v.value.get('type', "unspecified") == "unspecified": hasWarnings = True print( f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " + "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr) if hasDocBookErrors: print("Explanation: The documentation contains descriptions, examples, or defaults written in DocBook. " + "NixOS is in the process of migrating from DocBook to Markdown, and " + "DocBook is disallowed for in-tree modules. To change your contribution to "+ "use Markdown, apply mdDoc and literalMD. For example:\n" + "\n" + " example.foo = mkOption {\n" + " description = lib.mdDoc ''your description'';\n" + " defaultText = lib.literalMD ''your description of default'';\n" + " }\n" + "\n" + " example.enable = mkEnableOption (lib.mdDoc ''your thing'');", file = sys.stderr) if hasErrors: sys.exit(1) if hasWarnings and warningsAreErrors: print( "\x1b[1;31m" + "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " + "to false to ignore these warnings." + "\x1b[0m", file=sys.stderr) sys.exit(1) json.dump(convertMD(unpivot(options)), fp=sys.stdout)