clang/docs/tools/dump_format_style.py

   1 #!/usr/bin/env python3
   2 # A tool to parse the FormatStyle struct from Format.h and update the
   3 # documentation in ../ClangFormatStyleOptions.rst automatically.
   4 # Run from the directory in which this file is located to update the docs.
   5
   6 import argparse
   7 import inspect
   8 import os
   9 import re
  10 import sys
  11 from io import TextIOWrapper
  12 from typing import Set
  13
  14 CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")
  15 FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, "include/clang/Format/Format.h")
  16 INCLUDE_STYLE_FILE = os.path.join(
  17     CLANG_DIR, "include/clang/Tooling/Inclusions/IncludeStyle.h"
  18 )
  19 DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormatStyleOptions.rst")
  20
  21 PLURALS_FILE = os.path.join(os.path.dirname(__file__), "plurals.txt")
  22
  23 plurals: Set[str] = set()
  24 with open(PLURALS_FILE) as f:
  25     f.seek(0)
  26     plurals = set(f.read().splitlines())
  27
  28
  29 def substitute(text, tag, contents):
  30     replacement = "\n.. START_%s\n\n%s\n\n.. END_%s\n" % (tag, contents, tag)
  31     pattern = r"\n\.\. START_%s\n.*\n\.\. END_%s\n" % (tag, tag)
  32     return re.sub(pattern, "%s", text, flags=re.S) % replacement
  33
  34
  35 def register_plural(singular: str, plural: str):
  36     if plural not in plurals:
  37         if not hasattr(register_plural, "generated_new_plural"):
  38             print(
  39                 "Plural generation: you can use "
  40                 f"`git checkout -- {os.path.relpath(PLURALS_FILE)}` "
  41                 "to reemit warnings or `git add` to include new plurals\n"
  42             )
  43         register_plural.generated_new_plural = True
  44
  45         plurals.add(plural)
  46         with open(PLURALS_FILE, "a") as f:
  47             f.write(plural + "\n")
  48         cf = inspect.currentframe()
  49         lineno = ""
  50         if cf and cf.f_back:
  51             lineno = ":" + str(cf.f_back.f_lineno)
  52         print(
  53             f"{__file__}{lineno} check if plural of {singular} is {plural}",
  54             file=sys.stderr,
  55         )
  56     return plural
  57
  58
  59 def pluralize(word: str):
  60     lword = word.lower()
  61     if len(lword) >= 2 and lword[-1] == "y" and lword[-2] not in "aeiou":
  62         return register_plural(word, word[:-1] + "ies")
  63     elif lword.endswith(("s", "sh", "ch", "x", "z")):
  64         return register_plural(word, word[:-1] + "es")
  65     elif lword.endswith("fe"):
  66         return register_plural(word, word[:-2] + "ves")
  67     elif lword.endswith("f") and not lword.endswith("ff"):
  68         return register_plural(word, word[:-1] + "ves")
  69     else:
  70         return register_plural(word, word + "s")
  71
  72
  73 def to_yaml_type(typestr: str):
  74     if typestr == "bool":
  75         return "Boolean"
  76     elif typestr == "int":
  77         return "Integer"
  78     elif typestr == "unsigned":
  79         return "Unsigned"
  80     elif typestr == "std::string":
  81         return "String"
  82
  83     match = re.match(r"std::vector<(.*)>$", typestr)
  84     if match:
  85         return "List of " + pluralize(to_yaml_type(match.group(1)))
  86
  87     match = re.match(r"std::optional<(.*)>$", typestr)
  88     if match:
  89         return to_yaml_type(match.group(1))
  90
  91     return typestr
  92
  93
  94 def doxygen2rst(text):
  95     text = re.sub(r"<tt>\s*(.*?)\s*<\/tt>", r"``\1``", text)
  96     text = re.sub(r"\\c ([^ ,;\.]+)", r"``\1``", text)
  97     text = re.sub(r"\\\w+ ", "", text)
  98     return text
  99
 100
 101 def indent(text, columns, indent_first_line=True):
 102     indent_str = " " * columns
 103     s = re.sub(r"\n([^\n])", "\n" + indent_str + "\\1", text, flags=re.S)
 104     if not indent_first_line or s.startswith("\n"):
 105         return s
 106     return indent_str + s
 107
 108
 109 class Option(object):
 110     def __init__(self, name, opt_type, comment, version):
 111         self.name = name
 112         self.type = opt_type
 113         self.comment = comment.strip()
 114         self.enum = None
 115         self.nested_struct = None
 116         self.version = version
 117
 118     def __str__(self):
 119         s = ".. _%s:\n\n**%s** (``%s``) " % (
 120             self.name,
 121             self.name,
 122             to_yaml_type(self.type),
 123         )
 124         if self.version:
 125             s += ":versionbadge:`clang-format %s` " % self.version
 126         s += ":ref:`¶ <%s>`\n%s" % (self.name, doxygen2rst(indent(self.comment, 2)))
 127         if self.enum and self.enum.values:
 128             s += indent("\n\nPossible values:\n\n%s\n" % self.enum, 2)
 129         if self.nested_struct:
 130             s += indent(
 131                 "\n\nNested configuration flags:\n\n%s\n" % self.nested_struct, 2
 132             )
 133             s = s.replace("<option-name>", self.name)
 134         return s
 135
 136
 137 class NestedStruct(object):
 138     def __init__(self, name, comment):
 139         self.name = name
 140         self.comment = comment.strip()
 141         self.values = []
 142
 143     def __str__(self):
 144         return self.comment + "\n" + "\n".join(map(str, self.values))
 145
 146
 147 class NestedField(object):
 148     def __init__(self, name, comment, version):
 149         self.name = name
 150         self.comment = comment.strip()
 151         self.version = version
 152
 153     def __str__(self):
 154         if self.version:
 155             return "\n* ``%s`` :versionbadge:`clang-format %s`\n%s" % (
 156                 self.name,
 157                 self.version,
 158                 doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
 159             )
 160         return "\n* ``%s`` %s" % (
 161             self.name,
 162             doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
 163         )
 164
 165
 166 class Enum(object):
 167     def __init__(self, name, comment):
 168         self.name = name
 169         self.comment = comment.strip()
 170         self.values = []
 171
 172     def __str__(self):
 173         return "\n".join(map(str, self.values))
 174
 175
 176 class NestedEnum(object):
 177     def __init__(self, name, enumtype, comment, version, values):
 178         self.name = name
 179         self.comment = comment
 180         self.values = values
 181         self.type = enumtype
 182         self.version = version
 183
 184     def __str__(self):
 185         s = ""
 186         if self.version:
 187             s = "\n* ``%s %s`` :versionbadge:`clang-format %s`\n\n%s" % (
 188                 to_yaml_type(self.type),
 189                 self.name,
 190                 self.version,
 191                 doxygen2rst(indent(self.comment, 2)),
 192             )
 193         else:
 194             s = "\n* ``%s %s``\n%s" % (
 195                 to_yaml_type(self.type),
 196                 self.name,
 197                 doxygen2rst(indent(self.comment, 2)),
 198             )
 199         s += indent("\nPossible values:\n\n", 2)
 200         s += indent("\n".join(map(str, self.values)), 2)
 201         return s
 202
 203
 204 class EnumValue(object):
 205     def __init__(self, name, comment, config):
 206         self.name = name
 207         self.comment = comment
 208         self.config = config
 209
 210     def __str__(self):
 211         return "* ``%s`` (in configuration: ``%s``)\n%s" % (
 212             self.name,
 213             re.sub(".*_", "", self.config),
 214             doxygen2rst(indent(self.comment, 2)),
 215         )
 216
 217
 218 class OptionsReader:
 219     def __init__(self, header: TextIOWrapper):
 220         self.header = header
 221         self.in_code_block = False
 222         self.code_indent = 0
 223         self.lineno = 0
 224         self.last_err_lineno = -1
 225
 226     def __file_path(self):
 227         return os.path.relpath(self.header.name)
 228
 229     def __print_line(self, line: str):
 230         print(f"{self.lineno:>6} | {line}", file=sys.stderr)
 231
 232     def __warning(self, msg: str, line: str):
 233         print(f"{self.__file_path()}:{self.lineno}: warning: {msg}:", file=sys.stderr)
 234         self.__print_line(line)
 235
 236     def __clean_comment_line(self, line: str):
 237         match = re.match(r"^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$", line)
 238         if match:
 239             if self.in_code_block:
 240                 self.__warning("`\\code` in another `\\code`", line)
 241             self.in_code_block = True
 242             indent_str = match.group("indent")
 243             if not indent_str:
 244                 indent_str = ""
 245             self.code_indent = len(indent_str)
 246             lang = match.group("lang")
 247             if not lang:
 248                 lang = "c++"
 249             return f"\n{indent_str}.. code-block:: {lang}\n\n"
 250
 251         endcode_match = re.match(r"^/// +\\endcode$", line)
 252         if endcode_match:
 253             if not self.in_code_block:
 254                 self.__warning(
 255                     "no correct `\\code` found before this `\\endcode`", line
 256                 )
 257             self.in_code_block = False
 258             return ""
 259
 260         # check code block indentation
 261         if (
 262             self.in_code_block
 263             and not line == "///"
 264             and not line.startswith("///  " + " " * self.code_indent)
 265         ):
 266             if self.last_err_lineno == self.lineno - 1:
 267                 self.__print_line(line)
 268             else:
 269                 self.__warning("code block should be indented", line)
 270             self.last_err_lineno = self.lineno
 271
 272         match = re.match(r"^/// \\warning$", line)
 273         if match:
 274             return "\n.. warning::\n\n"
 275
 276         endwarning_match = re.match(r"^/// +\\endwarning$", line)
 277         if endwarning_match:
 278             return ""
 279
 280         match = re.match(r"^/// \\note$", line)
 281         if match:
 282             return "\n.. note::\n\n"
 283
 284         endnote_match = re.match(r"^/// +\\endnote$", line)
 285         if endnote_match:
 286             return ""
 287         return line[4:] + "\n"
 288
 289     def read_options(self):
 290         class State:
 291             (
 292                 BeforeStruct,
 293                 Finished,
 294                 InStruct,
 295                 InNestedStruct,
 296                 InNestedFieldComment,
 297                 InFieldComment,
 298                 InEnum,
 299                 InEnumMemberComment,
 300                 InNestedEnum,
 301                 InNestedEnumMemberComment,
 302             ) = range(10)
 303
 304         state = State.BeforeStruct
 305
 306         options = []
 307         enums = {}
 308         nested_structs = {}
 309         comment = ""
 310         enum = None
 311         nested_struct = None
 312         version = None
 313         deprecated = False
 314
 315         for line in self.header:
 316             self.lineno += 1
 317             line = line.strip()
 318             if state == State.BeforeStruct:
 319                 if line in ("struct FormatStyle {", "struct IncludeStyle {"):
 320                     state = State.InStruct
 321             elif state == State.InStruct:
 322                 if line.startswith("///"):
 323                     state = State.InFieldComment
 324                     comment = self.__clean_comment_line(line)
 325                 elif line == "};":
 326                     state = State.Finished
 327                     break
 328             elif state == State.InFieldComment:
 329                 if line.startswith(r"/// \version"):
 330                     match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
 331                     if match:
 332                         version = match.group("version")
 333                 elif line.startswith("/// @deprecated"):
 334                     deprecated = True
 335                 elif line.startswith("///"):
 336                     comment += self.__clean_comment_line(line)
 337                 elif line.startswith("enum"):
 338                     state = State.InEnum
 339                     name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
 340                     enum = Enum(name, comment)
 341                 elif line.startswith("struct"):
 342                     state = State.InNestedStruct
 343                     name = re.sub(r"struct\s+(\w+)\s*\{", "\\1", line)
 344                     nested_struct = NestedStruct(name, comment)
 345                 elif line.endswith(";"):
 346                     prefix = "// "
 347                     if line.startswith(prefix):
 348                         line = line[len(prefix) :]
 349                     state = State.InStruct
 350                     field_type, field_name = re.match(
 351                         r"([<>:\w(,\s)]+)\s+(\w+);", line
 352                     ).groups()
 353                     if deprecated:
 354                         field_type = "deprecated"
 355                         deprecated = False
 356
 357                     if not version:
 358                         self.__warning(f"missing version for {field_name}", line)
 359                     option = Option(str(field_name), str(field_type), comment, version)
 360                     options.append(option)
 361                     version = None
 362                 else:
 363                     raise Exception(
 364                         "Invalid format, expected comment, field or enum\n" + line
 365                     )
 366             elif state == State.InNestedStruct:
 367                 if line.startswith("///"):
 368                     state = State.InNestedFieldComment
 369                     comment = self.__clean_comment_line(line)
 370                 elif line == "};":
 371                     state = State.InStruct
 372                     nested_structs[nested_struct.name] = nested_struct
 373             elif state == State.InNestedFieldComment:
 374                 if line.startswith(r"/// \version"):
 375                     match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
 376                     if match:
 377                         version = match.group("version")
 378                 elif line.startswith("///"):
 379                     comment += self.__clean_comment_line(line)
 380                 elif line.startswith("enum"):
 381                     state = State.InNestedEnum
 382                     name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
 383                     enum = Enum(name, comment)
 384                 else:
 385                     state = State.InNestedStruct
 386                     field_type, field_name = re.match(
 387                         r"([<>:\w(,\s)]+)\s+(\w+);", line
 388                     ).groups()
 389                     # if not version:
 390                     #    self.__warning(f"missing version for {field_name}", line)
 391                     if field_type in enums:
 392                         nested_struct.values.append(
 393                             NestedEnum(
 394                                 field_name,
 395                                 field_type,
 396                                 comment,
 397                                 version,
 398                                 enums[field_type].values,
 399                             )
 400                         )
 401                     else:
 402                         nested_struct.values.append(
 403                             NestedField(field_type + " " + field_name, comment, version)
 404                         )
 405                     version = None
 406             elif state == State.InEnum:
 407                 if line.startswith("///"):
 408                     state = State.InEnumMemberComment
 409                     comment = self.__clean_comment_line(line)
 410                 elif line == "};":
 411                     state = State.InStruct
 412                     enums[enum.name] = enum
 413                 else:
 414                     # Enum member without documentation. Must be documented
 415                     # where the enum is used.
 416                     pass
 417             elif state == State.InNestedEnum:
 418                 if line.startswith("///"):
 419                     state = State.InNestedEnumMemberComment
 420                     comment = self.__clean_comment_line(line)
 421                 elif line == "};":
 422                     state = State.InNestedStruct
 423                     enums[enum.name] = enum
 424                 else:
 425                     # Enum member without documentation. Must be
 426                     # documented where the enum is used.
 427                     pass
 428             elif state == State.InEnumMemberComment:
 429                 if line.startswith("///"):
 430                     comment += self.__clean_comment_line(line)
 431                 else:
 432                     state = State.InEnum
 433                     val = line.replace(",", "")
 434                     pos = val.find(" // ")
 435                     if pos != -1:
 436                         config = val[pos + 4 :]
 437                         val = val[:pos]
 438                     else:
 439                         config = val
 440                     enum.values.append(EnumValue(val, comment, config))
 441             elif state == State.InNestedEnumMemberComment:
 442                 if line.startswith("///"):
 443                     comment += self.__clean_comment_line(line)
 444                 else:
 445                     state = State.InNestedEnum
 446                     val = line.replace(",", "")
 447                     pos = val.find(" // ")
 448                     if pos != -1:
 449                         config = val[pos + 4 :]
 450                         val = val[:pos]
 451                     else:
 452                         config = val
 453                     enum.values.append(EnumValue(val, comment, config))
 454         if state != State.Finished:
 455             raise Exception("Not finished by the end of file")
 456
 457         for option in options:
 458             if option.type not in [
 459                 "bool",
 460                 "unsigned",
 461                 "int",
 462                 "std::string",
 463                 "std::vector<std::string>",
 464                 "std::vector<IncludeCategory>",
 465                 "std::vector<RawStringFormat>",
 466                 "std::optional<unsigned>",
 467                 "deprecated",
 468             ]:
 469                 if option.type in enums:
 470                     option.enum = enums[option.type]
 471                 elif option.type in nested_structs:
 472                     option.nested_struct = nested_structs[option.type]
 473                 else:
 474                     raise Exception("Unknown type: %s" % option.type)
 475         return options
 476
 477
 478 p = argparse.ArgumentParser()
 479 p.add_argument("-o", "--output", help="path of output file")
 480 args = p.parse_args()
 481
 482 with open(FORMAT_STYLE_FILE) as f:
 483     opts = OptionsReader(f).read_options()
 484 with open(INCLUDE_STYLE_FILE) as f:
 485     opts += OptionsReader(f).read_options()
 486
 487 opts = sorted(opts, key=lambda x: x.name)
 488 options_text = "\n\n".join(map(str, opts))
 489
 490 with open(DOC_FILE, encoding="utf-8") as f:
 491     contents = f.read()
 492
 493 contents = substitute(contents, "FORMAT_STYLE_OPTIONS", options_text)
 494
 495 with open(
 496     args.output if args.output else DOC_FILE, "w", newline="", encoding="utf-8"
 497 ) as f:
 498     f.write(contents)