nixos/lib/make-options-doc/mergeJSON.py

   1 import collections
   2 import json
   3 import sys
   4 from typing import Any, Dict, List
   5
   6 # for MD conversion
   7 import mistune
   8 import re
   9 from xml.sax.saxutils import escape, quoteattr
  10
  11 JSON = Dict[str, Any]
  12
  13 class Key:
  14     def __init__(self, path: List[str]):
  15         self.path = path
  16     def __hash__(self):
  17         result = 0
  18         for id in self.path:
  19             result ^= hash(id)
  20         return result
  21     def __eq__(self, other):
  22         return type(self) is type(other) and self.path == other.path
  23
  24 Option = collections.namedtuple('Option', ['name', 'value'])
  25
  26 # pivot a dict of options keyed by their display name to a dict keyed by their path
  27 def pivot(options: Dict[str, JSON]) -> Dict[Key, Option]:
  28     result: Dict[Key, Option] = dict()
  29     for (name, opt) in options.items():
  30         result[Key(opt['loc'])] = Option(name, opt)
  31     return result
  32
  33 # pivot back to indexed-by-full-name
  34 # like the docbook build we'll just fail if multiple options with differing locs
  35 # render to the same option name.
  36 def unpivot(options: Dict[Key, Option]) -> Dict[str, JSON]:
  37     result: Dict[str, Dict] = dict()
  38     for (key, opt) in options.items():
  39         if opt.name in result:
  40             raise RuntimeError(
  41                 'multiple options with colliding ids found',
  42                 opt.name,
  43                 result[opt.name]['loc'],
  44                 opt.value['loc'],
  45             )
  46         result[opt.name] = opt.value
  47     return result
  48
  49 admonitions = {
  50     '.warning': 'warning',
  51     '.important': 'important',
  52     '.note': 'note'
  53 }
  54 class Renderer(mistune.renderers.BaseRenderer):
  55     def _get_method(self, name):
  56         try:
  57             return super(Renderer, self)._get_method(name)
  58         except AttributeError:
  59             def not_supported(*args, **kwargs):
  60                 raise NotImplementedError("md node not supported yet", name, args, **kwargs)
  61             return not_supported
  62
  63     def text(self, text):
  64         return escape(text)
  65     def paragraph(self, text):
  66         return text + "\n\n"
  67     def newline(self):
  68         return "<literallayout>\n</literallayout>"
  69     def codespan(self, text):
  70         return f"<literal>{escape(text)}</literal>"
  71     def block_code(self, text, info=None):
  72         info = f" language={quoteattr(info)}" if info is not None else ""
  73         return f"<programlisting{info}>\n{escape(text)}</programlisting>"
  74     def link(self, link, text=None, title=None):
  75         tag = "link"
  76         if link[0:1] == '#':
  77             if text == "":
  78                 tag = "xref"
  79             attr = "linkend"
  80             link = quoteattr(link[1:])
  81         else:
  82             # try to faithfully reproduce links that were of the form <link href="..."/>
  83             # in docbook format
  84             if text == link:
  85                 text = ""
  86             attr = "xlink:href"
  87             link = quoteattr(link)
  88         return f"<{tag} {attr}={link}>{text}</{tag}>"
  89     def list(self, text, ordered, level, start=None):
  90         if ordered:
  91             raise NotImplementedError("ordered lists not supported yet")
  92         return f"<itemizedlist>\n{text}\n</itemizedlist>"
  93     def list_item(self, text, level):
  94         return f"<listitem><para>{text}</para></listitem>\n"
  95     def block_text(self, text):
  96         return text
  97     def emphasis(self, text):
  98         return f"<emphasis>{text}</emphasis>"
  99     def strong(self, text):
 100         return f"<emphasis role=\"strong\">{text}</emphasis>"
 101     def admonition(self, text, kind):
 102         if kind not in admonitions:
 103             raise NotImplementedError(f"admonition {kind} not supported yet")
 104         tag = admonitions[kind]
 105         # we don't keep whitespace here because usually we'll contain only
 106         # a single paragraph and the original docbook string is no longer
 107         # available to restore the trailer.
 108         return f"<{tag}><para>{text.rstrip()}</para></{tag}>"
 109     def block_quote(self, text):
 110         return f"<blockquote><para>{text}</para></blockquote>"
 111     def command(self, text):
 112         return f"<command>{escape(text)}</command>"
 113     def option(self, text):
 114         return f"<option>{escape(text)}</option>"
 115     def file(self, text):
 116         return f"<filename>{escape(text)}</filename>"
 117     def var(self, text):
 118         return f"<varname>{escape(text)}</varname>"
 119     def env(self, text):
 120         return f"<envar>{escape(text)}</envar>"
 121     def manpage(self, page, section):
 122         title = f"<refentrytitle>{escape(page)}</refentrytitle>"
 123         vol = f"<manvolnum>{escape(section)}</manvolnum>"
 124         return f"<citerefentry>{title}{vol}</citerefentry>"
 125
 126     def finalize(self, data):
 127         return "".join(data)
 128
 129 def p_command(md):
 130     COMMAND_PATTERN = r'\{command\}`(.*?)`'
 131     def parse(self, m, state):
 132         return ('command', m.group(1))
 133     md.inline.register_rule('command', COMMAND_PATTERN, parse)
 134     md.inline.rules.append('command')
 135
 136 def p_file(md):
 137     FILE_PATTERN = r'\{file\}`(.*?)`'
 138     def parse(self, m, state):
 139         return ('file', m.group(1))
 140     md.inline.register_rule('file', FILE_PATTERN, parse)
 141     md.inline.rules.append('file')
 142
 143 def p_var(md):
 144     VAR_PATTERN = r'\{var\}`(.*?)`'
 145     def parse(self, m, state):
 146         return ('var', m.group(1))
 147     md.inline.register_rule('var', VAR_PATTERN, parse)
 148     md.inline.rules.append('var')
 149
 150 def p_env(md):
 151     ENV_PATTERN = r'\{env\}`(.*?)`'
 152     def parse(self, m, state):
 153         return ('env', m.group(1))
 154     md.inline.register_rule('env', ENV_PATTERN, parse)
 155     md.inline.rules.append('env')
 156
 157 def p_option(md):
 158     OPTION_PATTERN = r'\{option\}`(.*?)`'
 159     def parse(self, m, state):
 160         return ('option', m.group(1))
 161     md.inline.register_rule('option', OPTION_PATTERN, parse)
 162     md.inline.rules.append('option')
 163
 164 def p_manpage(md):
 165     MANPAGE_PATTERN = r'\{manpage\}`(.*?)\((.+?)\)`'
 166     def parse(self, m, state):
 167         return ('manpage', m.group(1), m.group(2))
 168     md.inline.register_rule('manpage', MANPAGE_PATTERN, parse)
 169     md.inline.rules.append('manpage')
 170
 171 def p_admonition(md):
 172     ADMONITION_PATTERN = re.compile(r'^::: \{([^\n]*?)\}\n(.*?)^:::$\n*', flags=re.MULTILINE|re.DOTALL)
 173     def parse(self, m, state):
 174         return {
 175             'type': 'admonition',
 176             'children': self.parse(m.group(2), state),
 177             'params': [ m.group(1) ],
 178         }
 179     md.block.register_rule('admonition', ADMONITION_PATTERN, parse)
 180     md.block.rules.append('admonition')
 181
 182 md = mistune.create_markdown(renderer=Renderer(), plugins=[
 183     p_command, p_file, p_var, p_env, p_option, p_manpage, p_admonition
 184 ])
 185
 186 # converts in-place!
 187 def convertMD(options: Dict[str, Any]) -> str:
 188     def convertString(path: str, text: str) -> str:
 189         try:
 190             rendered = md(text)
 191             # keep trailing spaces so we can diff the generated XML to check for conversion bugs.
 192             return rendered.rstrip() + text[len(text.rstrip()):]
 193         except:
 194             print(f"error in {path}")
 195             raise
 196
 197     def optionIs(option: Dict[str, Any], key: str, typ: str) -> bool:
 198         if key not in option: return False
 199         if type(option[key]) != dict: return False
 200         if '_type' not in option[key]: return False
 201         return option[key]['_type'] == typ
 202
 203     for (name, option) in options.items():
 204         if optionIs(option, 'description', 'mdDoc'):
 205             option['description'] = convertString(name, option['description']['text'])
 206         if optionIs(option, 'example', 'literalMD'):
 207             docbook = convertString(name, option['example']['text'])
 208             option['example'] = { '_type': 'literalDocBook', 'text': docbook }
 209         if optionIs(option, 'default', 'literalMD'):
 210             docbook = convertString(name, option['default']['text'])
 211             option['default'] = { '_type': 'literalDocBook', 'text': docbook }
 212
 213     return options
 214
 215 warningsAreErrors = False
 216 errorOnDocbook = False
 217 optOffset = 0
 218 for arg in sys.argv[1:]:
 219     if arg == "--warnings-are-errors":
 220         optOffset += 1
 221         warningsAreErrors = True
 222     if arg == "--error-on-docbook":
 223         optOffset += 1
 224         errorOnDocbook = True
 225
 226 options = pivot(json.load(open(sys.argv[1 + optOffset], 'r')))
 227 overrides = pivot(json.load(open(sys.argv[2 + optOffset], 'r')))
 228
 229 # fix up declaration paths in lazy options, since we don't eval them from a full nixpkgs dir
 230 for (k, v) in options.items():
 231     # The _module options are not declared in nixos/modules
 232     if v.value['loc'][0] != "_module":
 233         v.value['declarations'] = list(map(lambda s: f'nixos/modules/{s}' if isinstance(s, str) else s, v.value['declarations']))
 234
 235 # merge both descriptions
 236 for (k, v) in overrides.items():
 237     cur = options.setdefault(k, v).value
 238     for (ok, ov) in v.value.items():
 239         if ok == 'declarations':
 240             decls = cur[ok]
 241             for d in ov:
 242                 if d not in decls:
 243                     decls += [d]
 244         elif ok == "type":
 245             # ignore types of placeholder options
 246             if ov != "_unspecified" or cur[ok] == "_unspecified":
 247                 cur[ok] = ov
 248         elif ov is not None or cur.get(ok, None) is None:
 249             cur[ok] = ov
 250
 251 severity = "error" if warningsAreErrors else "warning"
 252
 253 def is_docbook(o, key):
 254     val = o.get(key, {})
 255     if not isinstance(val, dict):
 256         return False
 257     return val.get('_type', '') == 'literalDocBook'
 258
 259 # check that every option has a description
 260 hasWarnings = False
 261 hasErrors = False
 262 hasDocBookErrors = False
 263 for (k, v) in options.items():
 264     if errorOnDocbook:
 265         if isinstance(v.value.get('description', {}), str):
 266             hasErrors = True
 267             hasDocBookErrors = True
 268             print(
 269                 f"\x1b[1;31merror: option {v.name} description uses DocBook\x1b[0m",
 270                 file=sys.stderr)
 271         elif is_docbook(v.value, 'defaultText'):
 272             hasErrors = True
 273             hasDocBookErrors = True
 274             print(
 275                 f"\x1b[1;31merror: option {v.name} default uses DocBook\x1b[0m",
 276                 file=sys.stderr)
 277         elif is_docbook(v.value, 'example'):
 278             hasErrors = True
 279             hasDocBookErrors = True
 280             print(
 281                 f"\x1b[1;31merror: option {v.name} example uses DocBook\x1b[0m",
 282                 file=sys.stderr)
 283
 284     if v.value.get('description', None) is None:
 285         hasWarnings = True
 286         print(f"\x1b[1;31m{severity}: option {v.name} has no description\x1b[0m", file=sys.stderr)
 287         v.value['description'] = "This option has no description."
 288     if v.value.get('type', "unspecified") == "unspecified":
 289         hasWarnings = True
 290         print(
 291             f"\x1b[1;31m{severity}: option {v.name} has no type. Please specify a valid type, see " +
 292             "https://nixos.org/manual/nixos/stable/index.html#sec-option-types\x1b[0m", file=sys.stderr)
 293
 294 if hasDocBookErrors:
 295     print("Explanation: The documentation contains descriptions, examples, or defaults written in DocBook. " +
 296         "NixOS is in the process of migrating from DocBook to Markdown, and " +
 297         "DocBook is disallowed for in-tree modules. To change your contribution to "+
 298         "use Markdown, apply mdDoc and literalMD. For example:\n" +
 299         "\n" +
 300         "  example.foo = mkOption {\n" +
 301         "    description = lib.mdDoc ''your description'';\n" +
 302         "    defaultText = lib.literalMD ''your description of default'';\n" +
 303         "  }\n" +
 304         "\n" +
 305         "  example.enable = mkEnableOption (lib.mdDoc ''your thing'');",
 306         file = sys.stderr)
 307
 308 if hasErrors:
 309     sys.exit(1)
 310 if hasWarnings and warningsAreErrors:
 311     print(
 312         "\x1b[1;31m" +
 313         "Treating warnings as errors. Set documentation.nixos.options.warningsAreErrors " +
 314         "to false to ignore these warnings." +
 315         "\x1b[0m",
 316         file=sys.stderr)
 317     sys.exit(1)
 318
 319 json.dump(convertMD(unpivot(options)), fp=sys.stdout)