support/scripts/gen-manual-lists.py

   1 ## gen-manual-lists.py
   2 ##
   3 ## This script generates the following Buildroot manual appendices:
   4 ##  - the package tables (one for the target, the other for host tools);
   5 ##  - the deprecated items.
   6 ##
   7 ## Author(s):
   8 ##  - Samuel Martin <s.martin49@gmail.com>
   9 ##
  10 ## Copyright (C) 2013 Samuel Martin
  11 ##
  12 ## This program is free software; you can redistribute it and/or modify
  13 ## it under the terms of the GNU General Public License as published by
  14 ## the Free Software Foundation; either version 2 of the License, or
  15 ## (at your option) any later version.
  16 ##
  17 ## This program is distributed in the hope that it will be useful,
  18 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ## GNU General Public License for more details.
  21 ##
  22 ## You should have received a copy of the GNU General Public License
  23 ## along with this program; if not, write to the Free Software
  24 ## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  25 ##
  26
  27 from __future__ import print_function
  28 from __future__ import unicode_literals
  29
  30 import os
  31 import re
  32 import sys
  33 import datetime
  34 from argparse import ArgumentParser
  35
  36 try:
  37     import kconfiglib
  38 except ImportError:
  39     message = """
  40 Could not find the module 'kconfiglib' in the PYTHONPATH:
  41 """
  42     message += "\n".join(["  {0}".format(path) for path in sys.path])
  43     message += """
  44
  45 Make sure the Kconfiglib directory is in the PYTHONPATH, then relaunch the
  46 script.
  47
  48 You can get kconfiglib from:
  49   https://github.com/ulfalizer/Kconfiglib
  50
  51
  52 """
  53     sys.stderr.write(message)
  54     raise
  55
  56
  57 def get_symbol_subset(root, filter_func):
  58     """ Return a generator of kconfig items.
  59
  60     :param root_item:   Root item of the generated subset of items
  61     :param filter_func: Filter function
  62
  63     """
  64     if hasattr(root, "get_items"):
  65         get_items = root.get_items
  66     elif hasattr(root, "get_top_level_items"):
  67         get_items = root.get_top_level_items
  68     else:
  69         message = "The symbol does not contain any subset of symbols"
  70         raise Exception(message)
  71     for item in get_items():
  72         if item.is_symbol():
  73             if not filter_func(item):
  74                 continue
  75             yield item
  76         elif item.is_menu() or item.is_choice():
  77             for i in get_symbol_subset(item, filter_func):
  78                 yield i
  79
  80
  81 def get_symbol_parents(item, root=None, enable_choice=False):
  82     """ Return the list of the item's parents. The last item of the list is
  83     the closest parent, the first the furthest.
  84
  85     :param item:          Item from which the parent list is generated
  86     :param root:          Root item stopping the search (not included in the
  87                           parent list)
  88     :param enable_choice: Flag enabling choices to appear in the parent list
  89
  90     """
  91     parent = item.get_parent()
  92     parents = []
  93     while parent and parent != root:
  94         if parent.is_menu():
  95             parents.append(parent.get_title())
  96         elif enable_choice and parent.is_choice():
  97             parents.append(parent.get_prompts()[0])
  98         parent = parent.get_parent()
  99     if isinstance(root, kconfiglib.Menu) or \
 100             (enable_choice and isinstance(root, kconfiglib.Choice)):
 101         parents.append("") # Dummy empty parent to get a leading arrow ->
 102     parents.reverse()
 103     return parents
 104
 105
 106 def format_asciidoc_table(root, get_label_func, filter_func=lambda x: True,
 107                           format_func=lambda x: x,
 108                           enable_choice=False, sorted=True,
 109                           item_label=None):
 110     """ Return the asciidoc formatted table of the items and their location.
 111
 112     :param root:           Root item of the item subset
 113     :param get_label_func: Item's label getter function
 114     :param filter_func:    Filter function to apply on the item subset
 115     :param format_func:    Function to format a symbol and the table header
 116     :param enable_choice:  Enable choices to appear as part of the item's
 117                            location
 118     :param sorted:         Flag to alphabetically sort the table
 119
 120     """
 121
 122     lines = []
 123     for item in get_symbol_subset(root, filter_func):
 124         lines.append(format_func(what="symbol", symbol=item, root=root,
 125                                  get_label_func=get_label_func,
 126                                  enable_choice=enable_choice))
 127     if sorted:
 128         lines.sort(key=lambda x: x.lower())
 129     table = ":halign: center\n\n"
 130     width, columns = format_func(what="layout")
 131     table = "[width=\"{0}\",cols=\"{1}\",options=\"header\"]\n".format(width, columns)
 132     table += "|===================================================\n"
 133     table += format_func(what="header", header=item_label, root=root)
 134     table += "\n" + "".join(lines) + "\n"
 135     table += "|===================================================\n"
 136     return table
 137
 138
 139 class Buildroot:
 140     """ Buildroot configuration object.
 141
 142     """
 143     root_config = "Config.in"
 144     package_dirname = "package"
 145     package_prefixes = ["BR2_PACKAGE_", "BR2_PACKAGE_HOST_"]
 146     re_pkg_prefix = re.compile(r"^(" + "|".join(package_prefixes) + ").*")
 147     deprecated_symbol = "BR2_DEPRECATED"
 148     list_in = """\
 149 //
 150 // Automatically generated list for Buildroot manual.
 151 //
 152
 153 {table}
 154 """
 155
 156     list_info = {
 157         'target-packages': {
 158             'filename': "package-list",
 159             'root_menu': "Target packages",
 160             'filter': "_is_real_package",
 161             'format': "_format_symbol_prompt_location",
 162             'sorted': True,
 163         },
 164         'host-packages': {
 165             'filename': "host-package-list",
 166             'root_menu': "Host utilities",
 167             'filter': "_is_real_package",
 168             'format': "_format_symbol_prompt",
 169             'sorted': True,
 170         },
 171         'virtual-packages': {
 172             'filename': "virtual-package-list",
 173             'root_menu': "Target packages",
 174             'filter': "_is_virtual_package",
 175             'format': "_format_symbol_virtual",
 176             'sorted': True,
 177         },
 178         'deprecated': {
 179             'filename': "deprecated-list",
 180             'root_menu': None,
 181             'filter': "_is_deprecated_feature",
 182             'format': "_format_symbol_prompt_location",
 183             'sorted': False,
 184         },
 185     }
 186
 187     def __init__(self):
 188         self.base_dir = os.environ.get("TOPDIR")
 189         self.output_dir = os.environ.get("O")
 190         self.package_dir = os.path.join(self.base_dir, self.package_dirname)
 191         self.config = kconfiglib.Config(os.path.join(self.base_dir,
 192                                                      self.root_config),
 193                                         self.base_dir)
 194         self._deprecated = self.config.get_symbol(self.deprecated_symbol)
 195
 196         self.gen_date = datetime.datetime.utcnow()
 197         self.br_version_full = os.environ.get("BR2_VERSION_FULL")
 198         if self.br_version_full and self.br_version_full.endswith("-git"):
 199             self.br_version_full = self.br_version_full[:-4]
 200         if not self.br_version_full:
 201             self.br_version_full = "undefined"
 202
 203     def _get_package_symbols(self, package_name):
 204         """ Return a tuple containing the target and host package symbol.
 205
 206         """
 207         symbols = re.sub("[-+.]", "_", package_name)
 208         symbols = symbols.upper()
 209         symbols = tuple([prefix + symbols for prefix in self.package_prefixes])
 210         return symbols
 211
 212     def _is_deprecated(self, symbol):
 213         """ Return True if the symbol is marked as deprecated, otherwise False.
 214
 215         """
 216         # This also catches BR2_DEPRECATED_SINCE_xxxx_xx
 217         return bool([ symbol for x in symbol.get_referenced_symbols()
 218             if x.get_name().startswith(self._deprecated.get_name()) ])
 219
 220     def _is_package(self, symbol, type='real'):
 221         """ Return True if the symbol is a package or a host package, otherwise
 222         False.
 223
 224         :param symbol:  The symbol to check
 225         :param type:    Limit to 'real' or 'virtual' types of packages,
 226                         with 'real' being the default.
 227                         Note: only 'real' is (implictly) handled for now
 228
 229         """
 230         if not symbol.is_symbol():
 231             return False
 232         if type == 'real' and not symbol.get_prompts():
 233             return False
 234         if type == 'virtual' and symbol.get_prompts():
 235             return False
 236         if not self.re_pkg_prefix.match(symbol.get_name()):
 237             return False
 238         pkg_name = self._get_pkg_name(symbol)
 239
 240         pattern = "^(HOST_)?" + pkg_name + "$"
 241         pattern = re.sub("_", ".", pattern)
 242         pattern = re.compile(pattern, re.IGNORECASE)
 243         # Here, we cannot just check for the location of the Config.in because
 244         # of the "virtual" package.
 245         #
 246         # So, to check that a symbol is a package (not a package option or
 247         # anything else), we check for the existence of the package *.mk file.
 248         #
 249         # By the way, to actually check for a package, we should grep all *.mk
 250         # files for the following regex:
 251         # "\$\(eval \$\((host-)?(generic|autotools|cmake)-package\)\)"
 252         #
 253         # Implementation details:
 254         #
 255         # * The package list is generated from the *.mk file existence, the
 256         #   first time this function is called. Despite the memory consumption,
 257         #   this list is stored because the execution time of this script is
 258         #   noticeably shorter than rescanning the package sub-tree for each
 259         #   symbol.
 260         if not hasattr(self, "_package_list"):
 261             pkg_list = []
 262             for _, _, files in os.walk(self.package_dir):
 263                 for file_ in (f for f in files if f.endswith(".mk")):
 264                     pkg_list.append(re.sub(r"(.*?)\.mk", r"\1", file_))
 265             setattr(self, "_package_list", pkg_list)
 266         for pkg in getattr(self, "_package_list"):
 267             if type == 'real':
 268                 if pattern.match(pkg) and not self._exists_virt_symbol(pkg):
 269                     return True
 270             if type == 'virtual':
 271                 if pattern.match('has_' + pkg):
 272                     return True
 273         return False
 274
 275     def _is_real_package(self, symbol):
 276         return self._is_package(symbol, 'real')
 277
 278     def _is_virtual_package(self, symbol):
 279         return self._is_package(symbol, 'virtual')
 280
 281     def _is_deprecated_feature(self, symbol):
 282         return symbol.get_prompts() and self._is_deprecated(symbol)
 283
 284     def _exists_virt_symbol(self, pkg_name):
 285         """ Return True if a symbol exists that defines the package as
 286         a virtual package, False otherwise
 287
 288         :param pkg_name:    The name of the package, for which to check if
 289                             a symbol exists defining it as a virtual package
 290
 291         """
 292         virt_pattern = "BR2_PACKAGE_HAS_" + pkg_name + "$"
 293         virt_pattern = re.sub("_", ".", virt_pattern)
 294         virt_pattern = re.compile(virt_pattern, re.IGNORECASE)
 295         for sym in self.config:
 296             if virt_pattern.match(sym.get_name()):
 297                 return True
 298         return False
 299
 300     def _get_pkg_name(self, symbol):
 301         """ Return the package name of the specified symbol.
 302
 303         :param symbol:      The symbol to get the package name of
 304
 305         """
 306
 307         return re.sub("BR2_PACKAGE_(HOST_)?(.*)", r"\2", symbol.get_name())
 308
 309     def _get_symbol_label(self, symbol, mark_deprecated=True):
 310         """ Return the label (a.k.a. prompt text) of the symbol.
 311
 312         :param symbol:          The symbol
 313         :param mark_deprecated: Append a 'deprecated' to the label
 314
 315         """
 316         label = symbol.get_prompts()[0]
 317         if self._is_deprecated(symbol) and mark_deprecated:
 318             label += " *(deprecated)*"
 319         return label
 320
 321     def _format_symbol_prompt(self, what=None, symbol=None, root=None,
 322                                     enable_choice=False, header=None,
 323                                     get_label_func=lambda x: x):
 324         if what == "layout":
 325             return ( "30%", "^1" )
 326
 327         if what == "header":
 328             return "| {0:<40}\n".format(header)
 329
 330         if what == "symbol":
 331             return "| {0:<40}\n".format(get_label_func(symbol))
 332
 333         message = "Invalid argument 'what': '%s'\n" % str(what)
 334         message += "Allowed values are: 'layout', 'header' and 'symbol'"
 335         raise Exception(message)
 336
 337     def _format_symbol_prompt_location(self, what=None, symbol=None, root=None,
 338                                              enable_choice=False, header=None,
 339                                              get_label_func=lambda x: x):
 340         if what == "layout":
 341             return ( "100%", "^1,4" )
 342
 343         if what == "header":
 344             if hasattr(root, "get_title"):
 345                 loc_label = get_symbol_parents(root, None, enable_choice=enable_choice)
 346                 loc_label += [root.get_title(), "..."]
 347             else:
 348                 loc_label = ["Location"]
 349             return "| {0:<40} <| {1}\n".format(header, " -> ".join(loc_label))
 350
 351         if what == "symbol":
 352             parents = get_symbol_parents(symbol, root, enable_choice)
 353             return "| {0:<40} <| {1}\n".format(get_label_func(symbol),
 354                                                " -> ".join(parents))
 355
 356         message = "Invalid argument 'what': '%s'\n" % str(what)
 357         message += "Allowed values are: 'layout', 'header' and 'symbol'"
 358         raise Exception(message)
 359
 360     def _format_symbol_virtual(self, what=None, symbol=None, root=None,
 361                                      enable_choice=False, header=None,
 362                                      get_label_func=lambda x: "?"):
 363         def _symbol_is_legacy(symbol):
 364             selects = [ s.get_name() for s in symbol.get_selected_symbols() ]
 365             return ("BR2_LEGACY" in selects)
 366
 367         def _get_parent_package(sym):
 368             if self._is_real_package(sym):
 369                 return None
 370             # Trim the symbol name from its last component (separated with
 371             # underscores), until we either find a symbol which is a real
 372             # package, or until we have no component (i.e. just 'BR2')
 373             name = sym.get_name()
 374             while name != "BR2":
 375                 name = name.rsplit("_", 1)[0]
 376                 s = self.config.get_symbol(name)
 377                 if s is None:
 378                     continue
 379                 if self._is_real_package(s):
 380                     return s
 381             return None
 382
 383         def _get_providers(symbol):
 384             providers = list()
 385             for sym in self.config:
 386                 if not sym.is_symbol():
 387                     continue
 388                 if _symbol_is_legacy(sym):
 389                     continue
 390                 selects = sym.get_selected_symbols()
 391                 if not selects:
 392                     continue
 393                 for s in selects:
 394                     if s == symbol:
 395                         if sym.get_prompts():
 396                             l = self._get_symbol_label(sym,False)
 397                             parent_pkg = _get_parent_package(sym)
 398                             if parent_pkg is not None:
 399                                 l = self._get_symbol_label(parent_pkg, False) \
 400                                   + " (w/ " + l + ")"
 401                             providers.append(l)
 402                         else:
 403                             providers.extend(_get_providers(sym))
 404             return providers
 405
 406         if what == "layout":
 407             return ( "100%", "^1,4,4" )
 408
 409         if what == "header":
 410             return "| {0:<20} <| {1:<32} <| Providers\n".format("Virtual packages", "Symbols")
 411
 412         if what == "symbol":
 413             pkg = re.sub(r"^BR2_PACKAGE_HAS_(.+)$", r"\1", symbol.get_name())
 414             providers = _get_providers(symbol)
 415
 416             return "| {0:<20} <| {1:<32} <| {2}\n".format(pkg.lower(),
 417                                                           '+' + symbol.get_name() + '+',
 418                                                           ", ".join(providers))
 419
 420         message = "Invalid argument 'what': '%s'\n" % str(what)
 421         message += "Allowed values are: 'layout', 'header' and 'symbol'"
 422         raise Exception(message)
 423
 424
 425     def print_list(self, list_type, enable_choice=True, enable_deprecated=True,
 426                    dry_run=False, output=None):
 427         """ Print the requested list. If not dry run, then the list is
 428         automatically written in its own file.
 429
 430         :param list_type:         The list type to be generated
 431         :param enable_choice:     Flag enabling choices to appear in the list
 432         :param enable_deprecated: Flag enabling deprecated items to appear in
 433                                   the package lists
 434         :param dry_run:           Dry run (print the list in stdout instead of
 435                                   writing the list file
 436
 437         """
 438         def _get_menu(title):
 439             """ Return the first symbol menu matching the given title.
 440
 441             """
 442             menus = self.config.get_menus()
 443             menu = [m for m in menus if m.get_title().lower() == title.lower()]
 444             if not menu:
 445                 message = "No such menu: '{0}'".format(title)
 446                 raise Exception(message)
 447             return menu[0]
 448
 449         list_config = self.list_info[list_type]
 450         root_title = list_config.get('root_menu')
 451         if root_title:
 452             root_item = _get_menu(root_title)
 453         else:
 454             root_item = self.config
 455         filter_ = getattr(self, list_config.get('filter'))
 456         filter_func = lambda x: filter_(x)
 457         format_func = getattr(self, list_config.get('format'))
 458         if not enable_deprecated and list_type != "deprecated":
 459             filter_func = lambda x: filter_(x) and not self._is_deprecated(x)
 460         mark_depr = list_type != "deprecated"
 461         get_label = lambda x: self._get_symbol_label(x, mark_depr)
 462         item_label = "Features" if list_type == "deprecated" else "Packages"
 463
 464         table = format_asciidoc_table(root_item, get_label,
 465                                       filter_func=filter_func,
 466                                       format_func=format_func,
 467                                       enable_choice=enable_choice,
 468                                       sorted=list_config.get('sorted'),
 469                                       item_label=item_label)
 470
 471         content = self.list_in.format(table=table)
 472
 473         if dry_run:
 474             print(content)
 475             return
 476
 477         if not output:
 478             output_dir = self.output_dir
 479             if not output_dir:
 480                 print("Warning: Undefined output directory.")
 481                 print("\tUse source directory as output location.")
 482                 output_dir = self.base_dir
 483             output = os.path.join(output_dir,
 484                                   list_config.get('filename') + ".txt")
 485         if not os.path.exists(os.path.dirname(output)):
 486             os.makedirs(os.path.dirname(output))
 487         print("Writing the {0} list in:\n\t{1}".format(list_type, output))
 488         with open(output, 'w') as fout:
 489             fout.write(content)
 490
 491
 492 if __name__ == '__main__':
 493     list_types = ['target-packages', 'host-packages', 'virtual-packages', 'deprecated']
 494     parser = ArgumentParser()
 495     parser.add_argument("list_type", nargs="?", choices=list_types,
 496                         help="""\
 497 Generate the given list (generate all lists if unspecified)""")
 498     parser.add_argument("-n", "--dry-run", dest="dry_run", action='store_true',
 499                         help="Output the generated list to stdout")
 500     parser.add_argument("--output-target", dest="output_target",
 501                         help="Output target package file")
 502     parser.add_argument("--output-host", dest="output_host",
 503                         help="Output host package file")
 504     parser.add_argument("--output-virtual", dest="output_virtual",
 505                         help="Output virtual package file")
 506     parser.add_argument("--output-deprecated", dest="output_deprecated",
 507                         help="Output deprecated file")
 508     args = parser.parse_args()
 509     lists = [args.list_type] if args.list_type else list_types
 510     buildroot = Buildroot()
 511     for list_name in lists:
 512         output = getattr(args, "output_" + list_name.split("-", 1)[0])
 513         buildroot.print_list(list_name, dry_run=args.dry_run, output=output)