tools/net/ynl/ynl-gen-rst.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0
   3 # -*- coding: utf-8; mode: python -*-
   4
   5 """
   6     Script to auto generate the documentation for Netlink specifications.
   7
   8     :copyright:  Copyright (C) 2023  Breno Leitao <leitao@debian.org>
   9     :license:    GPL Version 2, June 1991 see linux/COPYING for details.
  10
  11     This script performs extensive parsing to the Linux kernel's netlink YAML
  12     spec files, in an effort to avoid needing to heavily mark up the original
  13     YAML file.
  14
  15     This code is split in three big parts:
  16         1) RST formatters: Use to convert a string to a RST output
  17         2) Parser helpers: Functions to parse the YAML data structure
  18         3) Main function and small helpers
  19 """
  20
  21 from typing import Any, Dict, List
  22 import os.path
  23 import sys
  24 import argparse
  25 import logging
  26 import yaml
  27
  28
  29 SPACE_PER_LEVEL = 4
  30
  31
  32 # RST Formatters
  33 # ==============
  34 def headroom(level: int) -> str:
  35     """Return space to format"""
  36     return " " * (level * SPACE_PER_LEVEL)
  37
  38
  39 def bold(text: str) -> str:
  40     """Format bold text"""
  41     return f"**{text}**"
  42
  43
  44 def inline(text: str) -> str:
  45     """Format inline text"""
  46     return f"``{text}``"
  47
  48
  49 def sanitize(text: str) -> str:
  50     """Remove newlines and multiple spaces"""
  51     # This is useful for some fields that are spread across multiple lines
  52     return str(text).replace("\n", " ").strip()
  53
  54
  55 def rst_fields(key: str, value: str, level: int = 0) -> str:
  56     """Return a RST formatted field"""
  57     return headroom(level) + f":{key}: {value}"
  58
  59
  60 def rst_definition(key: str, value: Any, level: int = 0) -> str:
  61     """Format a single rst definition"""
  62     return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
  63
  64
  65 def rst_paragraph(paragraph: str, level: int = 0) -> str:
  66     """Return a formatted paragraph"""
  67     return headroom(level) + paragraph
  68
  69
  70 def rst_bullet(item: str, level: int = 0) -> str:
  71     """Return a formatted a bullet"""
  72     return headroom(level) + f"- {item}"
  73
  74
  75 def rst_subsection(title: str) -> str:
  76     """Add a sub-section to the document"""
  77     return f"{title}\n" + "-" * len(title)
  78
  79
  80 def rst_subsubsection(title: str) -> str:
  81     """Add a sub-sub-section to the document"""
  82     return f"{title}\n" + "~" * len(title)
  83
  84
  85 def rst_section(namespace: str, prefix: str, title: str) -> str:
  86     """Add a section to the document"""
  87     return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title)
  88
  89
  90 def rst_subtitle(title: str) -> str:
  91     """Add a subtitle to the document"""
  92     return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
  93
  94
  95 def rst_title(title: str) -> str:
  96     """Add a title to the document"""
  97     return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
  98
  99
 100 def rst_list_inline(list_: List[str], level: int = 0) -> str:
 101     """Format a list using inlines"""
 102     return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
 103
 104
 105 def rst_ref(namespace: str, prefix: str, name: str) -> str:
 106     """Add a hyperlink to the document"""
 107     mappings = {'enum': 'definition',
 108                 'fixed-header': 'definition',
 109                 'nested-attributes': 'attribute-set',
 110                 'struct': 'definition'}
 111     if prefix in mappings:
 112         prefix = mappings[prefix]
 113     return f":ref:`{namespace}-{prefix}-{name}`"
 114
 115
 116 def rst_header() -> str:
 117     """The headers for all the auto generated RST files"""
 118     lines = []
 119
 120     lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
 121     lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
 122
 123     return "\n".join(lines)
 124
 125
 126 def rst_toctree(maxdepth: int = 2) -> str:
 127     """Generate a toctree RST primitive"""
 128     lines = []
 129
 130     lines.append(".. toctree::")
 131     lines.append(f"   :maxdepth: {maxdepth}\n\n")
 132
 133     return "\n".join(lines)
 134
 135
 136 def rst_label(title: str) -> str:
 137     """Return a formatted label"""
 138     return f".. _{title}:\n\n"
 139
 140
 141 # Parsers
 142 # =======
 143
 144
 145 def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
 146     """Parse 'multicast' group list and return a formatted string"""
 147     lines = []
 148     for group in mcast_group:
 149         lines.append(rst_bullet(group["name"]))
 150
 151     return "\n".join(lines)
 152
 153
 154 def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
 155     """Parse 'do' section and return a formatted string"""
 156     lines = []
 157     for key in do_dict.keys():
 158         lines.append(rst_paragraph(bold(key), level + 1))
 159         if key in ['request', 'reply']:
 160             lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
 161         else:
 162             lines.append(headroom(level + 2) + do_dict[key] + "\n")
 163
 164     return "\n".join(lines)
 165
 166
 167 def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
 168     """Parse 'attributes' section"""
 169     if "attributes" not in attrs:
 170         return ""
 171     lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
 172
 173     return "\n".join(lines)
 174
 175
 176 def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str:
 177     """Parse operations block"""
 178     preprocessed = ["name", "doc", "title", "do", "dump", "flags"]
 179     linkable = ["fixed-header", "attribute-set"]
 180     lines = []
 181
 182     for operation in operations:
 183         lines.append(rst_section(namespace, 'operation', operation["name"]))
 184         lines.append(rst_paragraph(operation["doc"]) + "\n")
 185
 186         for key in operation.keys():
 187             if key in preprocessed:
 188                 # Skip the special fields
 189                 continue
 190             value = operation[key]
 191             if key in linkable:
 192                 value = rst_ref(namespace, key, value)
 193             lines.append(rst_fields(key, value, 0))
 194         if 'flags' in operation:
 195             lines.append(rst_fields('flags', rst_list_inline(operation['flags'])))
 196
 197         if "do" in operation:
 198             lines.append(rst_paragraph(":do:", 0))
 199             lines.append(parse_do(operation["do"], 0))
 200         if "dump" in operation:
 201             lines.append(rst_paragraph(":dump:", 0))
 202             lines.append(parse_do(operation["dump"], 0))
 203
 204         # New line after fields
 205         lines.append("\n")
 206
 207     return "\n".join(lines)
 208
 209
 210 def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
 211     """Parse a list of entries"""
 212     ignored = ["pad"]
 213     lines = []
 214     for entry in entries:
 215         if isinstance(entry, dict):
 216             # entries could be a list or a dictionary
 217             field_name = entry.get("name", "")
 218             if field_name in ignored:
 219                 continue
 220             type_ = entry.get("type")
 221             if type_:
 222                 field_name += f" ({inline(type_)})"
 223             lines.append(
 224                 rst_fields(field_name, sanitize(entry.get("doc", "")), level)
 225             )
 226         elif isinstance(entry, list):
 227             lines.append(rst_list_inline(entry, level))
 228         else:
 229             lines.append(rst_bullet(inline(sanitize(entry)), level))
 230
 231     lines.append("\n")
 232     return "\n".join(lines)
 233
 234
 235 def parse_definitions(defs: Dict[str, Any], namespace: str) -> str:
 236     """Parse definitions section"""
 237     preprocessed = ["name", "entries", "members"]
 238     ignored = ["render-max"]  # This is not printed
 239     lines = []
 240
 241     for definition in defs:
 242         lines.append(rst_section(namespace, 'definition', definition["name"]))
 243         for k in definition.keys():
 244             if k in preprocessed + ignored:
 245                 continue
 246             lines.append(rst_fields(k, sanitize(definition[k]), 0))
 247
 248         # Field list needs to finish with a new line
 249         lines.append("\n")
 250         if "entries" in definition:
 251             lines.append(rst_paragraph(":entries:", 0))
 252             lines.append(parse_entries(definition["entries"], 1))
 253         if "members" in definition:
 254             lines.append(rst_paragraph(":members:", 0))
 255             lines.append(parse_entries(definition["members"], 1))
 256
 257     return "\n".join(lines)
 258
 259
 260 def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str:
 261     """Parse attribute from attribute-set"""
 262     preprocessed = ["name", "type"]
 263     linkable = ["enum", "nested-attributes", "struct", "sub-message"]
 264     ignored = ["checks"]
 265     lines = []
 266
 267     for entry in entries:
 268         lines.append(rst_section(namespace, 'attribute-set', entry["name"]))
 269         for attr in entry["attributes"]:
 270             type_ = attr.get("type")
 271             attr_line = attr["name"]
 272             if type_:
 273                 # Add the attribute type in the same line
 274                 attr_line += f" ({inline(type_)})"
 275
 276             lines.append(rst_subsubsection(attr_line))
 277
 278             for k in attr.keys():
 279                 if k in preprocessed + ignored:
 280                     continue
 281                 if k in linkable:
 282                     value = rst_ref(namespace, k, attr[k])
 283                 else:
 284                     value = sanitize(attr[k])
 285                 lines.append(rst_fields(k, value, 0))
 286             lines.append("\n")
 287
 288     return "\n".join(lines)
 289
 290
 291 def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str:
 292     """Parse sub-message definitions"""
 293     lines = []
 294
 295     for entry in entries:
 296         lines.append(rst_section(namespace, 'sub-message', entry["name"]))
 297         for fmt in entry["formats"]:
 298             value = fmt["value"]
 299
 300             lines.append(rst_bullet(bold(value)))
 301             for attr in ['fixed-header', 'attribute-set']:
 302                 if attr in fmt:
 303                     lines.append(rst_fields(attr,
 304                                             rst_ref(namespace, attr, fmt[attr]),
 305                                             1))
 306             lines.append("\n")
 307
 308     return "\n".join(lines)
 309
 310
 311 def parse_yaml(obj: Dict[str, Any]) -> str:
 312     """Format the whole YAML into a RST string"""
 313     lines = []
 314
 315     # Main header
 316
 317     lines.append(rst_header())
 318
 319     family = obj['name']
 320
 321     title = f"Family ``{family}`` netlink specification"
 322     lines.append(rst_title(title))
 323     lines.append(rst_paragraph(".. contents:: :depth: 3\n"))
 324
 325     if "doc" in obj:
 326         lines.append(rst_subtitle("Summary"))
 327         lines.append(rst_paragraph(obj["doc"], 0))
 328
 329     # Operations
 330     if "operations" in obj:
 331         lines.append(rst_subtitle("Operations"))
 332         lines.append(parse_operations(obj["operations"]["list"], family))
 333
 334     # Multicast groups
 335     if "mcast-groups" in obj:
 336         lines.append(rst_subtitle("Multicast groups"))
 337         lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
 338
 339     # Definitions
 340     if "definitions" in obj:
 341         lines.append(rst_subtitle("Definitions"))
 342         lines.append(parse_definitions(obj["definitions"], family))
 343
 344     # Attributes set
 345     if "attribute-sets" in obj:
 346         lines.append(rst_subtitle("Attribute sets"))
 347         lines.append(parse_attr_sets(obj["attribute-sets"], family))
 348
 349     # Sub-messages
 350     if "sub-messages" in obj:
 351         lines.append(rst_subtitle("Sub-messages"))
 352         lines.append(parse_sub_messages(obj["sub-messages"], family))
 353
 354     return "\n".join(lines)
 355
 356
 357 # Main functions
 358 # ==============
 359
 360
 361 def parse_arguments() -> argparse.Namespace:
 362     """Parse arguments from user"""
 363     parser = argparse.ArgumentParser(description="Netlink RST generator")
 364
 365     parser.add_argument("-v", "--verbose", action="store_true")
 366     parser.add_argument("-o", "--output", help="Output file name")
 367
 368     # Index and input are mutually exclusive
 369     group = parser.add_mutually_exclusive_group()
 370     group.add_argument(
 371         "-x", "--index", action="store_true", help="Generate the index page"
 372     )
 373     group.add_argument("-i", "--input", help="YAML file name")
 374
 375     args = parser.parse_args()
 376
 377     if args.verbose:
 378         logging.basicConfig(level=logging.DEBUG)
 379
 380     if args.input and not os.path.isfile(args.input):
 381         logging.warning("%s is not a valid file.", args.input)
 382         sys.exit(-1)
 383
 384     if not args.output:
 385         logging.error("No output file specified.")
 386         sys.exit(-1)
 387
 388     if os.path.isfile(args.output):
 389         logging.debug("%s already exists. Overwriting it.", args.output)
 390
 391     return args
 392
 393
 394 def parse_yaml_file(filename: str) -> str:
 395     """Transform the YAML specified by filename into a rst-formmated string"""
 396     with open(filename, "r", encoding="utf-8") as spec_file:
 397         yaml_data = yaml.safe_load(spec_file)
 398         content = parse_yaml(yaml_data)
 399
 400     return content
 401
 402
 403 def write_to_rstfile(content: str, filename: str) -> None:
 404     """Write the generated content into an RST file"""
 405     logging.debug("Saving RST file to %s", filename)
 406
 407     with open(filename, "w", encoding="utf-8") as rst_file:
 408         rst_file.write(content)
 409
 410
 411 def generate_main_index_rst(output: str) -> None:
 412     """Generate the `networking_spec/index` content and write to the file"""
 413     lines = []
 414
 415     lines.append(rst_header())
 416     lines.append(rst_label("specs"))
 417     lines.append(rst_title("Netlink Family Specifications"))
 418     lines.append(rst_toctree(1))
 419
 420     index_dir = os.path.dirname(output)
 421     logging.debug("Looking for .rst files in %s", index_dir)
 422     for filename in sorted(os.listdir(index_dir)):
 423         if not filename.endswith(".rst") or filename == "index.rst":
 424             continue
 425         lines.append(f"   {filename.replace('.rst', '')}\n")
 426
 427     logging.debug("Writing an index file at %s", output)
 428     write_to_rstfile("".join(lines), output)
 429
 430
 431 def main() -> None:
 432     """Main function that reads the YAML files and generates the RST files"""
 433
 434     args = parse_arguments()
 435
 436     if args.input:
 437         logging.debug("Parsing %s", args.input)
 438         try:
 439             content = parse_yaml_file(os.path.join(args.input))
 440         except Exception as exception:
 441             logging.warning("Failed to parse %s.", args.input)
 442             logging.warning(exception)
 443             sys.exit(-1)
 444
 445         write_to_rstfile(content, args.output)
 446
 447     if args.index:
 448         # Generate the index RST file
 449         generate_main_index_rst(args.output)
 450
 451
 452 if __name__ == "__main__":
 453     main()