clang/docs/tools/dump_format_style.py

   1 #!/usr/bin/env python3
   2 # A tool to parse the FormatStyle struct from Format.h and update the
   3 # documentation in ../ClangFormatStyleOptions.rst automatically.
   4 # Run from the directory in which this file is located to update the docs.
   5
   6 import inspect
   7 import os
   8 import re
   9 import sys
  10 from io import TextIOWrapper
  11 from typing import Set
  12
  13 CLANG_DIR = os.path.join(os.path.dirname(__file__), "../..")
  14 FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, "include/clang/Format/Format.h")
  15 INCLUDE_STYLE_FILE = os.path.join(
  16     CLANG_DIR, "include/clang/Tooling/Inclusions/IncludeStyle.h"
  17 )
  18 DOC_FILE = os.path.join(CLANG_DIR, "docs/ClangFormatStyleOptions.rst")
  19
  20 PLURALS_FILE = os.path.join(os.path.dirname(__file__), "plurals.txt")
  21
  22 plurals: Set[str] = set()
  23 with open(PLURALS_FILE, "a+") as f:
  24     f.seek(0)
  25     plurals = set(f.read().splitlines())
  26
  27
  28 def substitute(text, tag, contents):
  29     replacement = "\n.. START_%s\n\n%s\n\n.. END_%s\n" % (tag, contents, tag)
  30     pattern = r"\n\.\. START_%s\n.*\n\.\. END_%s\n" % (tag, tag)
  31     return re.sub(pattern, "%s", text, flags=re.S) % replacement
  32
  33
  34 def register_plural(singular: str, plural: str):
  35     if plural not in plurals:
  36         if not hasattr(register_plural, "generated_new_plural"):
  37             print(
  38                 "Plural generation: you can use "
  39                 f"`git checkout -- {os.path.relpath(PLURALS_FILE)}` "
  40                 "to reemit warnings or `git add` to include new plurals\n"
  41             )
  42         register_plural.generated_new_plural = True
  43
  44         plurals.add(plural)
  45         with open(PLURALS_FILE, "a") as f:
  46             f.write(plural + "\n")
  47         cf = inspect.currentframe()
  48         lineno = ""
  49         if cf and cf.f_back:
  50             lineno = ":" + str(cf.f_back.f_lineno)
  51         print(
  52             f"{__file__}{lineno} check if plural of {singular} is {plural}",
  53             file=sys.stderr,
  54         )
  55     return plural
  56
  57
  58 def pluralize(word: str):
  59     lword = word.lower()
  60     if len(lword) >= 2 and lword[-1] == "y" and lword[-2] not in "aeiou":
  61         return register_plural(word, word[:-1] + "ies")
  62     elif lword.endswith(("s", "sh", "ch", "x", "z")):
  63         return register_plural(word, word[:-1] + "es")
  64     elif lword.endswith("fe"):
  65         return register_plural(word, word[:-2] + "ves")
  66     elif lword.endswith("f") and not lword.endswith("ff"):
  67         return register_plural(word, word[:-1] + "ves")
  68     else:
  69         return register_plural(word, word + "s")
  70
  71
  72 def to_yaml_type(typestr: str):
  73     if typestr == "bool":
  74         return "Boolean"
  75     elif typestr == "int":
  76         return "Integer"
  77     elif typestr == "unsigned":
  78         return "Unsigned"
  79     elif typestr == "std::string":
  80         return "String"
  81
  82     match = re.match(r"std::vector<(.*)>$", typestr)
  83     if match:
  84         return "List of " + pluralize(to_yaml_type(match.group(1)))
  85
  86     match = re.match(r"std::optional<(.*)>$", typestr)
  87     if match:
  88         return to_yaml_type(match.group(1))
  89
  90     return typestr
  91
  92
  93 def doxygen2rst(text):
  94     text = re.sub(r"<tt>\s*(.*?)\s*<\/tt>", r"``\1``", text)
  95     text = re.sub(r"\\c ([^ ,;\.]+)", r"``\1``", text)
  96     text = re.sub(r"\\\w+ ", "", text)
  97     return text
  98
  99
 100 def indent(text, columns, indent_first_line=True):
 101     indent_str = " " * columns
 102     s = re.sub(r"\n([^\n])", "\n" + indent_str + "\\1", text, flags=re.S)
 103     if not indent_first_line or s.startswith("\n"):
 104         return s
 105     return indent_str + s
 106
 107
 108 class Option(object):
 109     def __init__(self, name, opt_type, comment, version):
 110         self.name = name
 111         self.type = opt_type
 112         self.comment = comment.strip()
 113         self.enum = None
 114         self.nested_struct = None
 115         self.version = version
 116
 117     def __str__(self):
 118         s = ".. _%s:\n\n**%s** (``%s``) " % (
 119             self.name,
 120             self.name,
 121             to_yaml_type(self.type),
 122         )
 123         if self.version:
 124             s += ":versionbadge:`clang-format %s` " % self.version
 125         s += ":ref:`¶ <%s>`\n%s" % (self.name, doxygen2rst(indent(self.comment, 2)))
 126         if self.enum and self.enum.values:
 127             s += indent("\n\nPossible values:\n\n%s\n" % self.enum, 2)
 128         if self.nested_struct:
 129             s += indent(
 130                 "\n\nNested configuration flags:\n\n%s\n" % self.nested_struct, 2
 131             )
 132         return s
 133
 134
 135 class NestedStruct(object):
 136     def __init__(self, name, comment):
 137         self.name = name
 138         self.comment = comment.strip()
 139         self.values = []
 140
 141     def __str__(self):
 142         return self.comment + "\n" + "\n".join(map(str, self.values))
 143
 144
 145 class NestedField(object):
 146     def __init__(self, name, comment, version):
 147         self.name = name
 148         self.comment = comment.strip()
 149         self.version = version
 150
 151     def __str__(self):
 152         if self.version:
 153             return "\n* ``%s`` :versionbadge:`clang-format %s`\n%s" % (
 154                 self.name,
 155                 self.version,
 156                 doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
 157             )
 158         return "\n* ``%s`` %s" % (
 159             self.name,
 160             doxygen2rst(indent(self.comment, 2, indent_first_line=False)),
 161         )
 162
 163
 164 class Enum(object):
 165     def __init__(self, name, comment):
 166         self.name = name
 167         self.comment = comment.strip()
 168         self.values = []
 169
 170     def __str__(self):
 171         return "\n".join(map(str, self.values))
 172
 173
 174 class NestedEnum(object):
 175     def __init__(self, name, enumtype, comment, version, values):
 176         self.name = name
 177         self.comment = comment
 178         self.values = values
 179         self.type = enumtype
 180         self.version = version
 181
 182     def __str__(self):
 183         s = ""
 184         if self.version:
 185             s = "\n* ``%s %s`` :versionbadge:`clang-format %s`\n\n%s" % (
 186                 to_yaml_type(self.type),
 187                 self.name,
 188                 self.version,
 189                 doxygen2rst(indent(self.comment, 2)),
 190             )
 191         else:
 192             s = "\n* ``%s %s``\n%s" % (
 193                 to_yaml_type(self.type),
 194                 self.name,
 195                 doxygen2rst(indent(self.comment, 2)),
 196             )
 197         s += indent("\nPossible values:\n\n", 2)
 198         s += indent("\n".join(map(str, self.values)), 2)
 199         return s
 200
 201
 202 class EnumValue(object):
 203     def __init__(self, name, comment, config):
 204         self.name = name
 205         self.comment = comment
 206         self.config = config
 207
 208     def __str__(self):
 209         return "* ``%s`` (in configuration: ``%s``)\n%s" % (
 210             self.name,
 211             re.sub(".*_", "", self.config),
 212             doxygen2rst(indent(self.comment, 2)),
 213         )
 214
 215
 216 class OptionsReader:
 217     def __init__(self, header: TextIOWrapper):
 218         self.header = header
 219         self.in_code_block = False
 220         self.code_indent = 0
 221         self.lineno = 0
 222         self.last_err_lineno = -1
 223
 224     def __file_path(self):
 225         return os.path.relpath(self.header.name)
 226
 227     def __print_line(self, line: str):
 228         print(f"{self.lineno:>6} | {line}", file=sys.stderr)
 229
 230     def __warning(self, msg: str, line: str):
 231         print(f"{self.__file_path()}:{self.lineno}: warning: {msg}:", file=sys.stderr)
 232         self.__print_line(line)
 233
 234     def __clean_comment_line(self, line: str):
 235         match = re.match(r"^/// (?P<indent> +)?\\code(\{.(?P<lang>\w+)\})?$", line)
 236         if match:
 237             if self.in_code_block:
 238                 self.__warning("`\\code` in another `\\code`", line)
 239             self.in_code_block = True
 240             indent_str = match.group("indent")
 241             if not indent_str:
 242                 indent_str = ""
 243             self.code_indent = len(indent_str)
 244             lang = match.group("lang")
 245             if not lang:
 246                 lang = "c++"
 247             return f"\n{indent_str}.. code-block:: {lang}\n\n"
 248
 249         endcode_match = re.match(r"^/// +\\endcode$", line)
 250         if endcode_match:
 251             if not self.in_code_block:
 252                 self.__warning(
 253                     "no correct `\\code` found before this `\\endcode`", line
 254                 )
 255             self.in_code_block = False
 256             return ""
 257
 258         # check code block indentation
 259         if (
 260             self.in_code_block
 261             and not line == "///"
 262             and not line.startswith("///  " + " " * self.code_indent)
 263         ):
 264             if self.last_err_lineno == self.lineno - 1:
 265                 self.__print_line(line)
 266             else:
 267                 self.__warning("code block should be indented", line)
 268             self.last_err_lineno = self.lineno
 269
 270         match = re.match(r"^/// \\warning$", line)
 271         if match:
 272             return "\n.. warning::\n\n"
 273
 274         endwarning_match = re.match(r"^/// +\\endwarning$", line)
 275         if endwarning_match:
 276             return ""
 277
 278         match = re.match(r"^/// \\note$", line)
 279         if match:
 280             return "\n.. note::\n\n"
 281
 282         endnote_match = re.match(r"^/// +\\endnote$", line)
 283         if endnote_match:
 284             return ""
 285         return line[4:] + "\n"
 286
 287     def read_options(self):
 288         class State:
 289             (
 290                 BeforeStruct,
 291                 Finished,
 292                 InStruct,
 293                 InNestedStruct,
 294                 InNestedFieldComment,
 295                 InFieldComment,
 296                 InEnum,
 297                 InEnumMemberComment,
 298                 InNestedEnum,
 299                 InNestedEnumMemberComment,
 300             ) = range(10)
 301
 302         state = State.BeforeStruct
 303
 304         options = []
 305         enums = {}
 306         nested_structs = {}
 307         comment = ""
 308         enum = None
 309         nested_struct = None
 310         version = None
 311
 312         for line in self.header:
 313             self.lineno += 1
 314             line = line.strip()
 315             if state == State.BeforeStruct:
 316                 if line in ("struct FormatStyle {", "struct IncludeStyle {"):
 317                     state = State.InStruct
 318             elif state == State.InStruct:
 319                 if line.startswith("///"):
 320                     state = State.InFieldComment
 321                     comment = self.__clean_comment_line(line)
 322                 elif line == "};":
 323                     state = State.Finished
 324                     break
 325             elif state == State.InFieldComment:
 326                 if line.startswith(r"/// \version"):
 327                     match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
 328                     if match:
 329                         version = match.group("version")
 330                 elif line.startswith("///"):
 331                     comment += self.__clean_comment_line(line)
 332                 elif line.startswith("enum"):
 333                     state = State.InEnum
 334                     name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
 335                     enum = Enum(name, comment)
 336                 elif line.startswith("struct"):
 337                     state = State.InNestedStruct
 338                     name = re.sub(r"struct\s+(\w+)\s*\{", "\\1", line)
 339                     nested_struct = NestedStruct(name, comment)
 340                 elif line.endswith(";"):
 341                     prefix = "// "
 342                     if line.startswith(prefix):
 343                         line = line[len(prefix) :]
 344                     state = State.InStruct
 345                     field_type, field_name = re.match(
 346                         r"([<>:\w(,\s)]+)\s+(\w+);", line
 347                     ).groups()
 348
 349                     if not version:
 350                         self.__warning(f"missing version for {field_name}", line)
 351                     option = Option(str(field_name), str(field_type), comment, version)
 352                     options.append(option)
 353                     version = None
 354                 else:
 355                     raise Exception(
 356                         "Invalid format, expected comment, field or enum\n" + line
 357                     )
 358             elif state == State.InNestedStruct:
 359                 if line.startswith("///"):
 360                     state = State.InNestedFieldComment
 361                     comment = self.__clean_comment_line(line)
 362                 elif line == "};":
 363                     state = State.InStruct
 364                     nested_structs[nested_struct.name] = nested_struct
 365             elif state == State.InNestedFieldComment:
 366                 if line.startswith(r"/// \version"):
 367                     match = re.match(r"/// \\version\s*(?P<version>[0-9.]+)*", line)
 368                     if match:
 369                         version = match.group("version")
 370                 elif line.startswith("///"):
 371                     comment += self.__clean_comment_line(line)
 372                 elif line.startswith("enum"):
 373                     state = State.InNestedEnum
 374                     name = re.sub(r"enum\s+(\w+)\s*(:((\s*\w+)+)\s*)?\{", "\\1", line)
 375                     enum = Enum(name, comment)
 376                 else:
 377                     state = State.InNestedStruct
 378                     field_type, field_name = re.match(
 379                         r"([<>:\w(,\s)]+)\s+(\w+);", line
 380                     ).groups()
 381                     # if not version:
 382                     #    self.__warning(f"missing version for {field_name}", line)
 383                     if field_type in enums:
 384                         nested_struct.values.append(
 385                             NestedEnum(
 386                                 field_name,
 387                                 field_type,
 388                                 comment,
 389                                 version,
 390                                 enums[field_type].values,
 391                             )
 392                         )
 393                     else:
 394                         nested_struct.values.append(
 395                             NestedField(field_type + " " + field_name, comment, version)
 396                         )
 397                     version = None
 398             elif state == State.InEnum:
 399                 if line.startswith("///"):
 400                     state = State.InEnumMemberComment
 401                     comment = self.__clean_comment_line(line)
 402                 elif line == "};":
 403                     state = State.InStruct
 404                     enums[enum.name] = enum
 405                 else:
 406                     # Enum member without documentation. Must be documented where the enum
 407                     # is used.
 408                     pass
 409             elif state == State.InNestedEnum:
 410                 if line.startswith("///"):
 411                     state = State.InNestedEnumMemberComment
 412                     comment = self.__clean_comment_line(line)
 413                 elif line == "};":
 414                     state = State.InNestedStruct
 415                     enums[enum.name] = enum
 416                 else:
 417                     # Enum member without documentation. Must be
 418                     # documented where the enum is used.
 419                     pass
 420             elif state == State.InEnumMemberComment:
 421                 if line.startswith("///"):
 422                     comment += self.__clean_comment_line(line)
 423                 else:
 424                     state = State.InEnum
 425                     val = line.replace(",", "")
 426                     pos = val.find(" // ")
 427                     if pos != -1:
 428                         config = val[pos + 4 :]
 429                         val = val[:pos]
 430                     else:
 431                         config = val
 432                     enum.values.append(EnumValue(val, comment, config))
 433             elif state == State.InNestedEnumMemberComment:
 434                 if line.startswith("///"):
 435                     comment += self.__clean_comment_line(line)
 436                 else:
 437                     state = State.InNestedEnum
 438                     val = line.replace(",", "")
 439                     pos = val.find(" // ")
 440                     if pos != -1:
 441                         config = val[pos + 4 :]
 442                         val = val[:pos]
 443                     else:
 444                         config = val
 445                     enum.values.append(EnumValue(val, comment, config))
 446         if state != State.Finished:
 447             raise Exception("Not finished by the end of file")
 448
 449         for option in options:
 450             if option.type not in [
 451                 "bool",
 452                 "unsigned",
 453                 "int",
 454                 "std::string",
 455                 "std::vector<std::string>",
 456                 "std::vector<IncludeCategory>",
 457                 "std::vector<RawStringFormat>",
 458                 "std::optional<unsigned>",
 459             ]:
 460                 if option.type in enums:
 461                     option.enum = enums[option.type]
 462                 elif option.type in nested_structs:
 463                     option.nested_struct = nested_structs[option.type]
 464                 else:
 465                     raise Exception("Unknown type: %s" % option.type)
 466         return options
 467
 468
 469 with open(FORMAT_STYLE_FILE) as f:
 470     opts = OptionsReader(f).read_options()
 471 with open(INCLUDE_STYLE_FILE) as f:
 472     opts += OptionsReader(f).read_options()
 473
 474 opts = sorted(opts, key=lambda x: x.name)
 475 options_text = "\n\n".join(map(str, opts))
 476
 477 with open(DOC_FILE) as f:
 478     contents = f.read()
 479
 480 contents = substitute(contents, "FORMAT_STYLE_OPTIONS", options_text)
 481
 482 with open(DOC_FILE, "wb") as output:
 483     output.write(contents.encode())