scripts/qapi/parser.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # QAPI schema parser
   4 #
   5 # Copyright IBM, Corp. 2011
   6 # Copyright (c) 2013-2019 Red Hat Inc.
   7 #
   8 # Authors:
   9 #  Anthony Liguori <aliguori@us.ibm.com>
  10 #  Markus Armbruster <armbru@redhat.com>
  11 #  Marc-André Lureau <marcandre.lureau@redhat.com>
  12 #  Kevin Wolf <kwolf@redhat.com>
  13 #
  14 # This work is licensed under the terms of the GNU GPL, version 2.
  15 # See the COPYING file in the top-level directory.
  16
  17 import os
  18 import re
  19 from collections import OrderedDict
  20
  21 from qapi.error import QAPIParseError, QAPISemError
  22 from qapi.source import QAPISourceInfo
  23
  24
  25 class QAPISchemaParser:
  26
  27     def __init__(self, fname, previously_included=None, incl_info=None):
  28         previously_included = previously_included or set()
  29         previously_included.add(os.path.abspath(fname))
  30
  31         try:
  32             fp = open(fname, 'r', encoding='utf-8')
  33             self.src = fp.read()
  34         except IOError as e:
  35             raise QAPISemError(incl_info or QAPISourceInfo(None, None, None),
  36                                "can't read %s file '%s': %s"
  37                                % ("include" if incl_info else "schema",
  38                                   fname,
  39                                   e.strerror))
  40
  41         if self.src == '' or self.src[-1] != '\n':
  42             self.src += '\n'
  43         self.cursor = 0
  44         self.info = QAPISourceInfo(fname, 1, incl_info)
  45         self.line_pos = 0
  46         self.exprs = []
  47         self.docs = []
  48         self.accept()
  49         cur_doc = None
  50
  51         while self.tok is not None:
  52             info = self.info
  53             if self.tok == '#':
  54                 self.reject_expr_doc(cur_doc)
  55                 cur_doc = self.get_doc(info)
  56                 self.docs.append(cur_doc)
  57                 continue
  58
  59             expr = self.get_expr(False)
  60             if 'include' in expr:
  61                 self.reject_expr_doc(cur_doc)
  62                 if len(expr) != 1:
  63                     raise QAPISemError(info, "invalid 'include' directive")
  64                 include = expr['include']
  65                 if not isinstance(include, str):
  66                     raise QAPISemError(info,
  67                                        "value of 'include' must be a string")
  68                 incl_fname = os.path.join(os.path.dirname(fname),
  69                                           include)
  70                 self.exprs.append({'expr': {'include': incl_fname},
  71                                    'info': info})
  72                 exprs_include = self._include(include, info, incl_fname,
  73                                               previously_included)
  74                 if exprs_include:
  75                     self.exprs.extend(exprs_include.exprs)
  76                     self.docs.extend(exprs_include.docs)
  77             elif "pragma" in expr:
  78                 self.reject_expr_doc(cur_doc)
  79                 if len(expr) != 1:
  80                     raise QAPISemError(info, "invalid 'pragma' directive")
  81                 pragma = expr['pragma']
  82                 if not isinstance(pragma, dict):
  83                     raise QAPISemError(
  84                         info, "value of 'pragma' must be an object")
  85                 for name, value in pragma.items():
  86                     self._pragma(name, value, info)
  87             else:
  88                 expr_elem = {'expr': expr,
  89                              'info': info}
  90                 if cur_doc:
  91                     if not cur_doc.symbol:
  92                         raise QAPISemError(
  93                             cur_doc.info, "definition documentation required")
  94                     expr_elem['doc'] = cur_doc
  95                 self.exprs.append(expr_elem)
  96             cur_doc = None
  97         self.reject_expr_doc(cur_doc)
  98
  99     @staticmethod
 100     def reject_expr_doc(doc):
 101         if doc and doc.symbol:
 102             raise QAPISemError(
 103                 doc.info,
 104                 "documentation for '%s' is not followed by the definition"
 105                 % doc.symbol)
 106
 107     def _include(self, include, info, incl_fname, previously_included):
 108         incl_abs_fname = os.path.abspath(incl_fname)
 109         # catch inclusion cycle
 110         inf = info
 111         while inf:
 112             if incl_abs_fname == os.path.abspath(inf.fname):
 113                 raise QAPISemError(info, "inclusion loop for %s" % include)
 114             inf = inf.parent
 115
 116         # skip multiple include of the same file
 117         if incl_abs_fname in previously_included:
 118             return None
 119
 120         return QAPISchemaParser(incl_fname, previously_included, info)
 121
 122     def _pragma(self, name, value, info):
 123         if name == 'doc-required':
 124             if not isinstance(value, bool):
 125                 raise QAPISemError(info,
 126                                    "pragma 'doc-required' must be boolean")
 127             info.pragma.doc_required = value
 128         elif name == 'returns-whitelist':
 129             if (not isinstance(value, list)
 130                     or any([not isinstance(elt, str) for elt in value])):
 131                 raise QAPISemError(
 132                     info,
 133                     "pragma returns-whitelist must be a list of strings")
 134             info.pragma.returns_whitelist = value
 135         elif name == 'name-case-whitelist':
 136             if (not isinstance(value, list)
 137                     or any([not isinstance(elt, str) for elt in value])):
 138                 raise QAPISemError(
 139                     info,
 140                     "pragma name-case-whitelist must be a list of strings")
 141             info.pragma.name_case_whitelist = value
 142         else:
 143             raise QAPISemError(info, "unknown pragma '%s'" % name)
 144
 145     def accept(self, skip_comment=True):
 146         while True:
 147             self.tok = self.src[self.cursor]
 148             self.pos = self.cursor
 149             self.cursor += 1
 150             self.val = None
 151
 152             if self.tok == '#':
 153                 if self.src[self.cursor] == '#':
 154                     # Start of doc comment
 155                     skip_comment = False
 156                 self.cursor = self.src.find('\n', self.cursor)
 157                 if not skip_comment:
 158                     self.val = self.src[self.pos:self.cursor]
 159                     return
 160             elif self.tok in '{}:,[]':
 161                 return
 162             elif self.tok == "'":
 163                 # Note: we accept only printable ASCII
 164                 string = ''
 165                 esc = False
 166                 while True:
 167                     ch = self.src[self.cursor]
 168                     self.cursor += 1
 169                     if ch == '\n':
 170                         raise QAPIParseError(self, "missing terminating \"'\"")
 171                     if esc:
 172                         # Note: we recognize only \\ because we have
 173                         # no use for funny characters in strings
 174                         if ch != '\\':
 175                             raise QAPIParseError(self,
 176                                                  "unknown escape \\%s" % ch)
 177                         esc = False
 178                     elif ch == '\\':
 179                         esc = True
 180                         continue
 181                     elif ch == "'":
 182                         self.val = string
 183                         return
 184                     if ord(ch) < 32 or ord(ch) >= 127:
 185                         raise QAPIParseError(
 186                             self, "funny character in string")
 187                     string += ch
 188             elif self.src.startswith('true', self.pos):
 189                 self.val = True
 190                 self.cursor += 3
 191                 return
 192             elif self.src.startswith('false', self.pos):
 193                 self.val = False
 194                 self.cursor += 4
 195                 return
 196             elif self.tok == '\n':
 197                 if self.cursor == len(self.src):
 198                     self.tok = None
 199                     return
 200                 self.info = self.info.next_line()
 201                 self.line_pos = self.cursor
 202             elif not self.tok.isspace():
 203                 # Show up to next structural, whitespace or quote
 204                 # character
 205                 match = re.match('[^[\\]{}:,\\s\'"]+',
 206                                  self.src[self.cursor-1:])
 207                 raise QAPIParseError(self, "stray '%s'" % match.group(0))
 208
 209     def get_members(self):
 210         expr = OrderedDict()
 211         if self.tok == '}':
 212             self.accept()
 213             return expr
 214         if self.tok != "'":
 215             raise QAPIParseError(self, "expected string or '}'")
 216         while True:
 217             key = self.val
 218             self.accept()
 219             if self.tok != ':':
 220                 raise QAPIParseError(self, "expected ':'")
 221             self.accept()
 222             if key in expr:
 223                 raise QAPIParseError(self, "duplicate key '%s'" % key)
 224             expr[key] = self.get_expr(True)
 225             if self.tok == '}':
 226                 self.accept()
 227                 return expr
 228             if self.tok != ',':
 229                 raise QAPIParseError(self, "expected ',' or '}'")
 230             self.accept()
 231             if self.tok != "'":
 232                 raise QAPIParseError(self, "expected string")
 233
 234     def get_values(self):
 235         expr = []
 236         if self.tok == ']':
 237             self.accept()
 238             return expr
 239         if self.tok not in "{['tfn":
 240             raise QAPIParseError(
 241                 self, "expected '{', '[', ']', string, boolean or 'null'")
 242         while True:
 243             expr.append(self.get_expr(True))
 244             if self.tok == ']':
 245                 self.accept()
 246                 return expr
 247             if self.tok != ',':
 248                 raise QAPIParseError(self, "expected ',' or ']'")
 249             self.accept()
 250
 251     def get_expr(self, nested):
 252         if self.tok != '{' and not nested:
 253             raise QAPIParseError(self, "expected '{'")
 254         if self.tok == '{':
 255             self.accept()
 256             expr = self.get_members()
 257         elif self.tok == '[':
 258             self.accept()
 259             expr = self.get_values()
 260         elif self.tok in "'tfn":
 261             expr = self.val
 262             self.accept()
 263         else:
 264             raise QAPIParseError(
 265                 self, "expected '{', '[', string, boolean or 'null'")
 266         return expr
 267
 268     def get_doc(self, info):
 269         if self.val != '##':
 270             raise QAPIParseError(
 271                 self, "junk after '##' at start of documentation comment")
 272
 273         doc = QAPIDoc(self, info)
 274         self.accept(False)
 275         while self.tok == '#':
 276             if self.val.startswith('##'):
 277                 # End of doc comment
 278                 if self.val != '##':
 279                     raise QAPIParseError(
 280                         self,
 281                         "junk after '##' at end of documentation comment")
 282                 doc.end_comment()
 283                 self.accept()
 284                 return doc
 285             doc.append(self.val)
 286             self.accept(False)
 287
 288         raise QAPIParseError(self, "documentation comment must end with '##'")
 289
 290
 291 class QAPIDoc:
 292     """
 293     A documentation comment block, either definition or free-form
 294
 295     Definition documentation blocks consist of
 296
 297     * a body section: one line naming the definition, followed by an
 298       overview (any number of lines)
 299
 300     * argument sections: a description of each argument (for commands
 301       and events) or member (for structs, unions and alternates)
 302
 303     * features sections: a description of each feature flag
 304
 305     * additional (non-argument) sections, possibly tagged
 306
 307     Free-form documentation blocks consist only of a body section.
 308     """
 309
 310     class Section:
 311         def __init__(self, name=None):
 312             # optional section name (argument/member or section name)
 313             self.name = name
 314             # the list of lines for this section
 315             self.text = ''
 316
 317         def append(self, line):
 318             self.text += line.rstrip() + '\n'
 319
 320     class ArgSection(Section):
 321         def __init__(self, name):
 322             super().__init__(name)
 323             self.member = None
 324
 325         def connect(self, member):
 326             self.member = member
 327
 328     def __init__(self, parser, info):
 329         # self._parser is used to report errors with QAPIParseError.  The
 330         # resulting error position depends on the state of the parser.
 331         # It happens to be the beginning of the comment.  More or less
 332         # servicable, but action at a distance.
 333         self._parser = parser
 334         self.info = info
 335         self.symbol = None
 336         self.body = QAPIDoc.Section()
 337         # dict mapping parameter name to ArgSection
 338         self.args = OrderedDict()
 339         self.features = OrderedDict()
 340         # a list of Section
 341         self.sections = []
 342         # the current section
 343         self._section = self.body
 344         self._append_line = self._append_body_line
 345
 346     def has_section(self, name):
 347         """Return True if we have a section with this name."""
 348         for i in self.sections:
 349             if i.name == name:
 350                 return True
 351         return False
 352
 353     def append(self, line):
 354         """
 355         Parse a comment line and add it to the documentation.
 356
 357         The way that the line is dealt with depends on which part of
 358         the documentation we're parsing right now:
 359         * The body section: ._append_line is ._append_body_line
 360         * An argument section: ._append_line is ._append_args_line
 361         * A features section: ._append_line is ._append_features_line
 362         * An additional section: ._append_line is ._append_various_line
 363         """
 364         line = line[1:]
 365         if not line:
 366             self._append_freeform(line)
 367             return
 368
 369         if line[0] != ' ':
 370             raise QAPIParseError(self._parser, "missing space after #")
 371         line = line[1:]
 372         self._append_line(line)
 373
 374     def end_comment(self):
 375         self._end_section()
 376
 377     @staticmethod
 378     def _is_section_tag(name):
 379         return name in ('Returns:', 'Since:',
 380                         # those are often singular or plural
 381                         'Note:', 'Notes:',
 382                         'Example:', 'Examples:',
 383                         'TODO:')
 384
 385     def _append_body_line(self, line):
 386         """
 387         Process a line of documentation text in the body section.
 388
 389         If this a symbol line and it is the section's first line, this
 390         is a definition documentation block for that symbol.
 391
 392         If it's a definition documentation block, another symbol line
 393         begins the argument section for the argument named by it, and
 394         a section tag begins an additional section.  Start that
 395         section and append the line to it.
 396
 397         Else, append the line to the current section.
 398         """
 399         name = line.split(' ', 1)[0]
 400         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 401         # recognized, and get silently treated as ordinary text
 402         if not self.symbol and not self.body.text and line.startswith('@'):
 403             if not line.endswith(':'):
 404                 raise QAPIParseError(self._parser, "line should end with ':'")
 405             self.symbol = line[1:-1]
 406             # FIXME invalid names other than the empty string aren't flagged
 407             if not self.symbol:
 408                 raise QAPIParseError(self._parser, "invalid name")
 409         elif self.symbol:
 410             # This is a definition documentation block
 411             if name.startswith('@') and name.endswith(':'):
 412                 self._append_line = self._append_args_line
 413                 self._append_args_line(line)
 414             elif line == 'Features:':
 415                 self._append_line = self._append_features_line
 416             elif self._is_section_tag(name):
 417                 self._append_line = self._append_various_line
 418                 self._append_various_line(line)
 419             else:
 420                 self._append_freeform(line.strip())
 421         else:
 422             # This is a free-form documentation block
 423             self._append_freeform(line.strip())
 424
 425     def _append_args_line(self, line):
 426         """
 427         Process a line of documentation text in an argument section.
 428
 429         A symbol line begins the next argument section, a section tag
 430         section or a non-indented line after a blank line begins an
 431         additional section.  Start that section and append the line to
 432         it.
 433
 434         Else, append the line to the current section.
 435
 436         """
 437         name = line.split(' ', 1)[0]
 438
 439         if name.startswith('@') and name.endswith(':'):
 440             line = line[len(name)+1:]
 441             self._start_args_section(name[1:-1])
 442         elif self._is_section_tag(name):
 443             self._append_line = self._append_various_line
 444             self._append_various_line(line)
 445             return
 446         elif (self._section.text.endswith('\n\n')
 447               and line and not line[0].isspace()):
 448             if line == 'Features:':
 449                 self._append_line = self._append_features_line
 450             else:
 451                 self._start_section()
 452                 self._append_line = self._append_various_line
 453                 self._append_various_line(line)
 454             return
 455
 456         self._append_freeform(line.strip())
 457
 458     def _append_features_line(self, line):
 459         name = line.split(' ', 1)[0]
 460
 461         if name.startswith('@') and name.endswith(':'):
 462             line = line[len(name)+1:]
 463             self._start_features_section(name[1:-1])
 464         elif self._is_section_tag(name):
 465             self._append_line = self._append_various_line
 466             self._append_various_line(line)
 467             return
 468         elif (self._section.text.endswith('\n\n')
 469               and line and not line[0].isspace()):
 470             self._start_section()
 471             self._append_line = self._append_various_line
 472             self._append_various_line(line)
 473             return
 474
 475         self._append_freeform(line.strip())
 476
 477     def _append_various_line(self, line):
 478         """
 479         Process a line of documentation text in an additional section.
 480
 481         A symbol line is an error.
 482
 483         A section tag begins an additional section.  Start that
 484         section and append the line to it.
 485
 486         Else, append the line to the current section.
 487         """
 488         name = line.split(' ', 1)[0]
 489
 490         if name.startswith('@') and name.endswith(':'):
 491             raise QAPIParseError(self._parser,
 492                                  "'%s' can't follow '%s' section"
 493                                  % (name, self.sections[0].name))
 494         if self._is_section_tag(name):
 495             line = line[len(name)+1:]
 496             self._start_section(name[:-1])
 497
 498         if (not self._section.name or
 499                 not self._section.name.startswith('Example')):
 500             line = line.strip()
 501
 502         self._append_freeform(line)
 503
 504     def _start_symbol_section(self, symbols_dict, name):
 505         # FIXME invalid names other than the empty string aren't flagged
 506         if not name:
 507             raise QAPIParseError(self._parser, "invalid parameter name")
 508         if name in symbols_dict:
 509             raise QAPIParseError(self._parser,
 510                                  "'%s' parameter name duplicated" % name)
 511         assert not self.sections
 512         self._end_section()
 513         self._section = QAPIDoc.ArgSection(name)
 514         symbols_dict[name] = self._section
 515
 516     def _start_args_section(self, name):
 517         self._start_symbol_section(self.args, name)
 518
 519     def _start_features_section(self, name):
 520         self._start_symbol_section(self.features, name)
 521
 522     def _start_section(self, name=None):
 523         if name in ('Returns', 'Since') and self.has_section(name):
 524             raise QAPIParseError(self._parser,
 525                                  "duplicated '%s' section" % name)
 526         self._end_section()
 527         self._section = QAPIDoc.Section(name)
 528         self.sections.append(self._section)
 529
 530     def _end_section(self):
 531         if self._section:
 532             text = self._section.text = self._section.text.strip()
 533             if self._section.name and (not text or text.isspace()):
 534                 raise QAPIParseError(
 535                     self._parser,
 536                     "empty doc section '%s'" % self._section.name)
 537             self._section = None
 538
 539     def _append_freeform(self, line):
 540         match = re.match(r'(@\S+:)', line)
 541         if match:
 542             raise QAPIParseError(self._parser,
 543                                  "'%s' not allowed in free-form documentation"
 544                                  % match.group(1))
 545         self._section.append(line)
 546
 547     def connect_member(self, member):
 548         if member.name not in self.args:
 549             # Undocumented TODO outlaw
 550             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 551         self.args[member.name].connect(member)
 552
 553     def connect_feature(self, feature):
 554         if feature.name not in self.features:
 555             raise QAPISemError(feature.info,
 556                                "feature '%s' lacks documentation"
 557                                % feature.name)
 558         self.features[feature.name].connect(feature)
 559
 560     def check_expr(self, expr):
 561         if self.has_section('Returns') and 'command' not in expr:
 562             raise QAPISemError(self.info,
 563                                "'Returns:' is only valid for commands")
 564
 565     def check(self):
 566
 567         def check_args_section(args, info, what):
 568             bogus = [name for name, section in args.items()
 569                      if not section.member]
 570             if bogus:
 571                 raise QAPISemError(
 572                     self.info,
 573                     "documented member%s '%s' %s not exist"
 574                     % ("s" if len(bogus) > 1 else "",
 575                        "', '".join(bogus),
 576                        "do" if len(bogus) > 1 else "does"))
 577
 578         check_args_section(self.args, self.info, 'members')
 579         check_args_section(self.features, self.info, 'features')