bin/update_pch

   1 #! /usr/bin/env python
   2 # -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10
  11 """
  12 This script generates precompiled headers for a given
  13 module and library.
  14
  15 Given a gmake makefile that belongs to some LO module:
  16 1) Process the makefile to find source files (process_makefile).
  17 2) For every source file, find all includes (process_source).
  18 3) Uncommon and rare includes are filtered (remove_rare).
  19 4) Conflicting headers are excluded (filter_ignore).
  20 5) Local files to the source are excluded (Filter_Local).
  21 6) Fixup missing headers that sources expect (fixup).
  22 7) The resulting includes are sorted by category (sort_by_category).
  23 8) The pch file is generated (generate).
  24 """
  25
  26 import sys
  27 import re
  28 import os
  29 import unittest
  30
  31 CUTOFF = 1
  32 EXCLUDE_MODULE = False
  33 EXCLUDE_LOCAL = False
  34 EXCLUDE_SYSTEM = True
  35 SILENT = False
  36
  37 # System includes: oox, sal, sd, svl, vcl
  38
  39 INCLUDE = False
  40 EXCLUDE = True
  41 DEFAULTS = \
  42 {
  43 #    module.library : (min, system, module, local), best time
  44     'accessibility.acc'                 : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   7.8
  45     'basctl.basctl'                     : ( 3, EXCLUDE, INCLUDE, EXCLUDE), #  11.9
  46     'basegfx.basegfx'                   : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   3.8
  47     'basic.sb'                          : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #  10.7
  48     'chart2.chartcontroller'            : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  18.4
  49     'chart2.chartcore'                  : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #  22.5
  50     'chart2.chartopengl'                : (12, EXCLUDE, EXCLUDE, EXCLUDE), #   5.3
  51     'comphelper.comphelper'             : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   7.6
  52     'configmgr.configmgr'               : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   6.0
  53     'connectivity.ado'                  : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   6.4
  54     'connectivity.calc'                 : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
  55     'connectivity.dbase'                : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   5.2
  56     'connectivity.dbpool2'              : ( 5, EXCLUDE, INCLUDE, EXCLUDE), #   3.0
  57     'connectivity.dbtools'              : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   0.8
  58     'connectivity.file'                 : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   5.1
  59     'connectivity.firebird_sdbc'        : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   5.1
  60     'connectivity.flat'                 : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.6
  61     'connectivity.mysql'                : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #   3.4
  62     'connectivity.odbc'                 : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   5.0
  63     'connectivity.postgresql-sdbc-impl' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   6.7
  64     'cppcanvas.cppcanvas'               : (11, EXCLUDE, INCLUDE, INCLUDE), #   4.8
  65     'cppuhelper.cppuhelper'             : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
  66     'cui.cui'                           : ( 8, EXCLUDE, INCLUDE, EXCLUDE), #  19.7
  67     'dbaccess.dba'                      : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  13.8
  68     'dbaccess.dbaxml'                   : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   6.5
  69     'dbaccess.dbmm'                     : (10, EXCLUDE, INCLUDE, EXCLUDE), #   4.3
  70     'dbaccess.dbu'                      : (12, EXCLUDE, EXCLUDE, EXCLUDE), #  23.6
  71     'dbaccess.sdbt'                     : ( 1, EXCLUDE, INCLUDE, EXCLUDE), #   2.9
  72     'desktop.deployment'                : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   6.1
  73     'desktop.deploymentgui'             : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   5.7
  74     'desktop.deploymentmisc'            : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   3.4
  75     'desktop.sofficeapp'                : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   6.5
  76     'drawinglayer.drawinglayer'         : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), #   7.4
  77     'editeng.editeng'                   : ( 5, EXCLUDE, INCLUDE, EXCLUDE), #  13.0
  78     'forms.frm'                         : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #  14.2
  79     'framework.fwe'                     : (10, EXCLUDE, INCLUDE, EXCLUDE), #   5.5
  80     'framework.fwi'                     : ( 9, EXCLUDE, INCLUDE, EXCLUDE), #   3.4
  81     'framework.fwk'                     : ( 7, EXCLUDE, INCLUDE, INCLUDE), #  14.8
  82     'framework.fwl'                     : ( 5, EXCLUDE, INCLUDE, INCLUDE), #   5.1
  83     'hwpfilter.hwp'                     : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   6.0
  84     'lotuswordpro.lwpft'                : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #  11.6
  85     'oox.oox'                           : ( 6, EXCLUDE, EXCLUDE, INCLUDE), #  28.2
  86     'package.package2'                  : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   4.5
  87     'package.xstor'                     : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   3.8
  88     'reportdesign.rpt'                  : ( 9, EXCLUDE, INCLUDE, INCLUDE), #   9.4
  89     'reportdesign.rptui'                : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  13.1
  90     'reportdesign.rptxml'               : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   7.6
  91     'sal.sal'                           : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   4.2
  92     'sc.sc'                             : (12, EXCLUDE, INCLUDE, INCLUDE), #  92.6
  93     'sc.scfilt'                         : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #  39.9
  94     'sc.scui'                           : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #  15.0
  95     'sc.vbaobj'                         : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #  17.3
  96     'sd.sd'                             : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #  47.4
  97     'sd.sdui'                           : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   9.4
  98     'sdext.PresentationMinimizer'       : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.1
  99     'sdext.PresenterScreen'             : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   7.1
 100     'sfx2.sfx'                          : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #  27.4
 101     'slideshow.slideshow'               : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #  10.8
 102     'sot.sot'                           : ( 5, EXCLUDE, EXCLUDE, INCLUDE), #   3.1
 103     'starmath.sm'                       : ( 5, EXCLUDE, EXCLUDE, INCLUDE), #  10.9
 104     'svgio.svgio'                       : ( 8, EXCLUDE, EXCLUDE, INCLUDE), #   4.3
 105     'emfio.emfio'                       : ( 8, EXCLUDE, EXCLUDE, INCLUDE), #   4.3
 106     'svl.svl'                           : ( 6, EXCLUDE, EXCLUDE, EXCLUDE), #   7.6
 107     'svtools.svt'                       : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #  17.6
 108     'svx.svx'                           : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #  20.7
 109     'svx.svxcore'                       : ( 7, EXCLUDE, INCLUDE, EXCLUDE), #  37.0
 110     'sw.msword'                         : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  22.4
 111     'sw.sw'                             : ( 7, EXCLUDE, EXCLUDE, INCLUDE), # 129.6
 112     'sw.swui'                           : ( 3, EXCLUDE, INCLUDE, INCLUDE), #  26.1
 113     'sw.vbaswobj'                       : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  13.1
 114     'tools.tl'                          : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), #   4.2
 115     'unotools.utl'                      : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   7.0
 116     'unoxml.unoxml'                     : ( 1, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
 117     'uui.uui'                           : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), #   4.9
 118     'vbahelper.msforms'                 : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   5.2
 119     'vbahelper.vbahelper'               : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   7.0
 120     'vcl.vcl'                           : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  35.7
 121     'writerfilter.writerfilter'         : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), #  19.7/27.3
 122     'xmloff.xo'                         : ( 7, EXCLUDE, INCLUDE, INCLUDE), #  22.1
 123     'xmloff.xof'                        : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #   4.4
 124     'xmlscript.xmlscript'               : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #   3.6
 125     'xmlsecurity.xmlsecurity'           : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   5.1
 126     'xmlsecurity.xsec_xmlsec'           : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.4
 127     'xmlsecurity.xsec_gpg'              : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   ?
 128 }
 129
 130 def remove_rare(raw, min_use=-1):
 131     """ Remove headers not commonly included.
 132         The minimum threshold is min_use.
 133     """
 134     # The minimum number of times a header
 135     # must be included to be in the PCH.
 136     min_use = min_use if min_use >= 0 else CUTOFF
 137
 138     out = []
 139     if not raw or not len(raw):
 140         return out
 141
 142     inc = sorted(raw)
 143     last = inc[0]
 144     count = 1
 145     for x in range(1, len(inc)):
 146         i = inc[x]
 147         if i == last:
 148             count += 1
 149         else:
 150             if count >= min_use:
 151                 out.append(last)
 152             last = i
 153             count = 1
 154
 155     # Last group.
 156     if count >= min_use:
 157         out.append(last)
 158
 159     return out
 160
 161 def process_list(list, callable):
 162     """ Given a list and callable
 163         we pass each entry through
 164         the callable and only add to
 165         the output if not blank.
 166     """
 167     out = []
 168     for i in list:
 169         line = callable(i)
 170         if line and len(line):
 171             out.append(line)
 172     return out
 173
 174 def find_files(path, recurse=True):
 175     list = []
 176     for root, dir, files in os.walk(path):
 177         list += map(lambda x: os.path.join(root, x), files)
 178     return list
 179
 180 def get_filename(line):
 181     """ Strips the line from the
 182         '#include' and angled brakets
 183         and return the filename only.
 184     """
 185     if not len(line) or line[0] != '#':
 186         return line
 187     return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line)
 188
 189 def is_c_runtime(inc):
 190     """ Heuristic-based detection of C/C++
 191         runtime headers.
 192         They are all-lowercase, with .h or
 193         no extension, filename only.
 194     """
 195     inc = get_filename(inc)
 196
 197     if inc.endswith('.hxx') or inc.endswith('.hpp'):
 198         return False
 199
 200     for c in inc:
 201         if c == '/':
 202             return False
 203         if c == '.':
 204             return inc.endswith('.h')
 205         if c.isupper():
 206             return False
 207
 208     return True
 209
 210 def sanitize(raw):
 211     """ There are two forms of includes,
 212         those with <> and "".
 213         Technically, the difference is that
 214         the compiler can use an internal
 215         representation for an angled include,
 216         such that it doesn't have to be a file.
 217         For our purposes, there is no difference.
 218         Here, we convert everything to angled.
 219     """
 220     if not raw or not len(raw):
 221         return ''
 222     raw = raw.strip()
 223     if not len(raw):
 224         return ''
 225     return re.sub(r'(.*#include\s*)\"(.*)\"(.*)', r'#include <\2>', raw)
 226
 227 class Filter_Local(object):
 228     """ Filter headers local to a module.
 229         allow_public: allows include/module/file.hxx
 230                       #include <module/file.hxx>
 231         allow_module: allows module/inc/file.hxx
 232                       #include <file.hxx>
 233         allow_locals: allows module/source/file.hxx and
 234                              module/source/inc/file.hxx
 235                       #include <file.hxx>
 236     """
 237     def __init__(self, root, module, allow_public=True, allow_module=True, allow_locals=True):
 238         self.root = root
 239         self.module = module
 240         self.allow_public = allow_public
 241         self.allow_module = allow_module
 242         self.allow_locals = allow_locals
 243         self.public_prefix = '<' + self.module + '/'
 244
 245         all = find_files(os.path.join(root, module))
 246         self.module = []
 247         self.locals = []
 248         mod_prefix = module + '/inc/'
 249         for i in all:
 250             if mod_prefix in i:
 251                 self.module.append(i)
 252             else:
 253                 self.locals.append(i)
 254
 255     def is_public(self, line):
 256         return self.public_prefix in line
 257
 258     def is_module(self, line):
 259         """ Returns True if in module/inc/... """
 260         filename = get_filename(line)
 261         for i in self.module:
 262             if i.endswith(filename):
 263                 return True
 264         return False
 265
 266     def is_local(self, line):
 267         """ Returns True if in module/source/... """
 268         filename = get_filename(line)
 269         for i in self.locals:
 270             if i.endswith(filename):
 271                 return True
 272         return False
 273
 274     def is_external(self, line):
 275         return is_c_runtime(line) and \
 276                not self.is_public(line) and \
 277                not self.is_module(line) and \
 278                not self.is_local(line)
 279
 280     def find_local_file(self, line):
 281         """ Finds the header file in the module dir,
 282             but doesn't validate.
 283         """
 284         filename = get_filename(line)
 285         for i in self.locals:
 286             if i.endswith(filename):
 287                 return i
 288         for i in self.module:
 289             if i.endswith(filename):
 290                 return i
 291         return None
 292
 293     def proc(self, line):
 294         assert line and len(line)
 295         assert line[0] != '<' and line[0] != '#'
 296
 297         filename = get_filename(line)
 298
 299         # Local with relative path.
 300         if filename.startswith('..'):
 301             # Exclude for now as we don't have cxx path.
 302             return ''
 303
 304         # Locals are included first (by the compiler).
 305         if self.is_local(filename):
 306             return line if self.allow_locals and '/inc/' in filename else ''
 307
 308         # Module headers are next.
 309         if self.is_module(filename):
 310             return line if self.allow_module else ''
 311
 312         # Public headers are last.
 313         if self.is_public(line):
 314             return line if self.allow_public else ''
 315
 316         # Leave out potentially unrelated files local
 317         # to some other module we can't include directly.
 318         if '/' not in filename and not self.is_external(filename):
 319             return ''
 320
 321         # Unfiltered.
 322         return line
 323
 324 def filter_ignore(line, module):
 325     """ Filters includes from known
 326         problematic ones.
 327         Expects sanitized input.
 328     """
 329     assert line and len(line)
 330
 331     # Always include files without extension.
 332     if '.' not in line:
 333         return line
 334
 335     # Extract filenames for ease of comparison.
 336     line = get_filename(line)
 337
 338     # Filter out all files that are not normal headers.
 339     if not line.endswith('.h') and \
 340        not line.endswith('.hxx') and \
 341        not line.endswith('.hpp') and \
 342        not line.endswith('.hdl'):
 343        return ''
 344
 345     ignore_list = [
 346             'LibreOfficeKit/LibreOfficeKitEnums.h', # Needs special directives
 347             'LibreOfficeKit/LibreOfficeKitTypes.h', # Needs special directives
 348             'jerror.h',     # c++ unfriendly
 349             'jpeglib.h',    # c++ unfriendly
 350             'boost/spirit/include/classic_core.hpp', # depends on BOOST_SPIRIT_DEBUG
 351             'svtools/editimplementation.hxx' # no direct include
 352         ]
 353
 354     if module == 'accessibility':
 355         ignore_list += [
 356             # STR_SVT_ACC_LISTENTRY_SELCTED_STATE redefined from svtools.hrc
 357             'accessibility/extended/textwindowaccessibility.hxx',
 358             ]
 359     if module == 'basic':
 360         ignore_list += [
 361             'basic/vbahelper.hxx',
 362             ]
 363     if module == 'connectivity':
 364         ignore_list += [
 365             'com/sun/star/beans/PropertyAttribute.hpp', # OPTIONAL defined via objbase.h
 366             'com/sun/star/sdbcx/Privilege.hpp', # DELETE defined via objbase.h
 367             ]
 368     if module == 'sc':
 369         ignore_list += [
 370             'progress.hxx', # special directives
 371             'scslots.hxx',  # special directives
 372            ]
 373     if module == 'sd':
 374         ignore_list += [
 375             'sdgslots.hxx', # special directives
 376             'sdslots.hxx',  # special directives
 377            ]
 378     if module == 'sfx2':
 379         ignore_list += [
 380             'sfx2/recentdocsview.hxx', # Redefines ApplicationType defined in objidl.h
 381             'sfx2/sidebar/Sidebar.hxx',
 382             'sfx2/sidebar/UnoSidebar.hxx',
 383             'sfxslots.hxx', # externally defined types
 384             ]
 385     if module == 'sot':
 386         ignore_list += [
 387             'sysformats.hxx',   # Windows headers
 388             ]
 389     if module == 'vcl':
 390         ignore_list += [
 391             'accmgr.hxx',   # redefines ImplAccelList
 392             'image.h',
 393             'jobset.h',
 394             'opengl/gdiimpl.hxx',
 395             'opengl/salbmp.hxx',
 396             'openglgdiimpl',   # ReplaceTextA
 397             'printdlg.hxx',
 398             'salinst.hxx',  # GetDefaultPrinterA
 399             'salprn.hxx',   # SetPrinterDataA
 400             'vcl/jobset.hxx',
 401             'vcl/oldprintadaptor.hxx',
 402             'vcl/opengl/OpenGLContext.hxx',
 403             'vcl/opengl/OpenGLHelper.hxx',  # Conflicts with X header on *ix
 404             'vcl/print.hxx',
 405             'vcl/prntypes.hxx', # redefines Orientation from filter/jpeg/Exif.hxx
 406             'vcl/sysdata.hxx',
 407             ]
 408     if module == 'xmloff':
 409         ignore_list += [
 410             'SchXMLExport.hxx', # SchXMLAutoStylePoolP.hxx not found
 411             'SchXMLImport.hxx', # enums redefined in draw\sdxmlimp_impl.hxx
 412             'XMLEventImportHelper.hxx', # NameMap redefined in XMLEventExport.hxx
 413             'xmloff/XMLEventExport.hxx', # enums redefined
 414             ]
 415     if module == 'xmlsecurity':
 416         ignore_list += [
 417             'xmlsec/*',
 418             'xmlsecurity/xmlsec-wrapper.h',
 419             ]
 420
 421     for i in ignore_list:
 422         if line.startswith(i):
 423             return ''
 424         if i[0] == '*' and line.endswith(i[1:]):
 425             return ''
 426         if i[-1] == '*' and line.startswith(i[:-1]):
 427             return ''
 428
 429     return line
 430
 431 def fixup(includes, module):
 432     """ Here we add any headers
 433         necessary in the pch.
 434         These could be known to be very
 435         common but for technical reasons
 436         left out of the pch by this generator.
 437         Or, they could be missing from the
 438         source files where they are used
 439         (probably because they had been
 440         in the old pch, they were missed).
 441         Also, these could be headers
 442         that make the build faster but
 443         aren't added automatically.
 444     """
 445     fixes = []
 446     def append(inc):
 447         # Add a space to exclude from
 448         # ignore bisecting.
 449         line = ' #include <{}>'.format(inc)
 450         try:
 451             i = fixes.index(inc)
 452             fixes[i] = inc
 453         except:
 454             fixes.append(inc)
 455
 456     if module == 'basctl':
 457         if 'basslots.hxx' in includes:
 458             append('sfx2/msg.hxx')
 459
 460     #if module == 'sc':
 461     #    if 'scslots.hxx' in includes:
 462     #        append('sfx2/msg.hxx')
 463     return fixes
 464
 465 def sort_by_category(list, module, filter_local):
 466     """ Move all 'system' headers first.
 467         Core files of osl, rtl, sal, next.
 468         Everything non-module-specific third.
 469         Last, module-specific headers.
 470     """
 471     sys = []
 472     boo = []
 473     cor = []
 474     rst = []
 475     mod = []
 476
 477     prefix = '<' + module + '/'
 478     for i in list:
 479         if is_c_runtime(i):
 480             sys.append(i)
 481         elif '<boost/' in i:
 482             boo.append(i)
 483         elif '<osl' in i or '<rtl' in i or '<sal' in i or '<vcl' in i:
 484             cor.append(i)
 485         elif prefix in i:
 486             mod.append(i)
 487         else:
 488             rst.append(i)
 489
 490     out = []
 491     out += sorted(sys)
 492     out += sorted(boo)
 493     out += sorted(cor)
 494     out += sorted(rst)
 495     out += sorted(mod)
 496     return out
 497
 498 def parse_makefile(groups, lines, lineno, lastif, ifstack):
 499
 500     inobjects = False
 501     inelse = False
 502     os_cond_re = re.compile('(ifeq|ifneq)\s*\(\$\(OS\)\,(\w*)\)')
 503
 504     line = lines[lineno]
 505     if line.startswith('if'):
 506         lastif = line
 507         if ifstack == 0:
 508             # Correction if first line is an if.
 509             lineno = parse_makefile(groups, lines, lineno, line, ifstack+1)
 510     else:
 511         lineno -= 1
 512
 513     while lineno + 1 < len(lines):
 514         lineno += 1
 515         line = lines[lineno].strip()
 516         line = line.rstrip('\\').strip()
 517         #print('line #{}: {}'.format(lineno, line))
 518         if len(line) == 0:
 519             continue
 520
 521         if line == '))':
 522             inobjects = False
 523         elif 'add_exception_objects' in line or \
 524              'add_cxxobject' in line:
 525              inobjects = True
 526              #print('inobjects')
 527              #if ifstack and not SILENT:
 528                 #sys.stderr.write('Sources in a conditional, ignoring for now.\n')
 529         elif line.startswith('if'):
 530             lineno = parse_makefile(groups, lines, lineno, line, ifstack+1)
 531             continue
 532         elif line.startswith('endif'):
 533             if ifstack:
 534                 return lineno
 535             continue
 536         elif line.startswith('else'):
 537             inelse = True
 538         elif inobjects:
 539             if EXCLUDE_SYSTEM and ifstack:
 540                 continue
 541             file = line + '.cxx'
 542             if ',' in line or '(' in line or ')' in line:
 543                 #print('passing: ' + line)
 544                 pass # $if() probably, or something similar
 545             else:
 546                 osname = ''
 547                 if lastif:
 548                     if 'filter' in lastif:
 549                         # We can't grok filter, yet.
 550                         continue
 551                     match = os_cond_re.match(lastif)
 552                     if not match:
 553                         # We only support OS conditionals.
 554                         continue
 555                     in_out = match.group(1)
 556                     osname = match.group(2) if match else ''
 557                     if (in_out == 'ifneq' and not inelse) or \
 558                        (in_out == 'ifeq' and inelse):
 559                         osname = '!' + osname
 560
 561                 if osname not in groups:
 562                     groups[osname] = []
 563                 groups[osname].append(file)
 564
 565     return groups
 566
 567 def process_makefile(root, module, makefile):
 568     """ Parse a gmake makefile and extract
 569         source filenames from it.
 570     """
 571
 572     filename = os.path.join(os.path.join(root, module), makefile)
 573     if not os.path.isfile(filename):
 574         sys.stderr.write('Error: Module {} has no makefile at {}.'.format(module, filename))
 575
 576     groups = {'':[], 'ANDROID':[], 'IOS':[], 'WNT':[], 'LINUX':[], 'MACOSX':[]}
 577
 578     with open(filename, 'r') as f:
 579         lines = f.readlines()
 580         groups = parse_makefile(groups, lines, lineno=0, lastif=None, ifstack=0)
 581
 582     return groups
 583
 584 def process_source(root, module, filename, maxdepth=0):
 585     """ Process a source file to extract
 586         included headers.
 587         For now, skip on compiler directives.
 588         maxdepth is used when processing headers
 589         which typically have protecting ifndef.
 590     """
 591
 592     ifdepth = 0
 593     lastif = ''
 594     raw_includes = []
 595     with open(filename, 'r') as f:
 596         for line in f:
 597             line = line.strip()
 598             if line.startswith('#if'):
 599                 ifdepth += 1
 600                 lastif = line
 601             elif line.startswith('#endif'):
 602                 ifdepth -= 1
 603                 lastif = '#if'
 604             elif line.startswith('#include'):
 605                 if ifdepth <= maxdepth:
 606                     line = sanitize(line)
 607                     if line:
 608                         line = get_filename(line)
 609                     if line and len(line):
 610                         raw_includes.append(line)
 611                 elif not SILENT:
 612                     sys.stderr.write('#include in {} : {}\n'.format(lastif, line))
 613
 614     return raw_includes
 615
 616 def explode(root, module, includes, tree, filter_local, recurse):
 617     incpath = os.path.join(root, 'include')
 618
 619     for inc in includes:
 620         filename = get_filename(inc)
 621         if filename in tree or len(filter_local.proc(filename)) == 0:
 622             continue
 623
 624         try:
 625             # Module or Local header.
 626             filepath = filter_local.find_local_file(inc)
 627             if filepath:
 628                 #print('trying loc: ' + filepath)
 629                 incs = process_source(root, module, filepath, maxdepth=1)
 630                 incs = map(get_filename, incs)
 631                 incs = process_list(incs, lambda x: filter_ignore(x, module))
 632                 incs = process_list(incs, filter_local.proc)
 633                 tree[filename] = incs
 634                 if recurse:
 635                     tree = explode(root, module, incs, tree, filter_local, recurse)
 636                 #print('{} => {}'.format(filepath, tree[filename]))
 637                 continue
 638         except:
 639             pass
 640
 641         try:
 642             # Public header.
 643             filepath = os.path.join(incpath, filename)
 644             #print('trying pub: ' + filepath)
 645             incs = process_source(root, module, filepath, maxdepth=1)
 646             incs = map(get_filename, incs)
 647             incs = process_list(incs, lambda x: filter_ignore(x, module))
 648             incs = process_list(incs, filter_local.proc)
 649             tree[filename] = incs
 650             if recurse:
 651                 tree = explode(root, module, incs, tree, filter_local, recurse)
 652             #print('{} => {}'.format(filepath, tree[filename]))
 653             continue
 654         except:
 655             pass
 656
 657         # Failed, but remember to avoid searching again.
 658         tree[filename] = []
 659
 660     return tree
 661
 662 def make_command_line():
 663     args = sys.argv[:]
 664     # Remove command line flags and
 665     # use internal flags.
 666     for i in xrange(len(args)-1, 0, -1):
 667         if args[i].startswith('--'):
 668             args.pop(i)
 669
 670     args.append('--cutoff=' + str(CUTOFF))
 671     if EXCLUDE_SYSTEM:
 672         args.append('--exclude:system')
 673     else:
 674         args.append('--include:system')
 675     if EXCLUDE_MODULE:
 676         args.append('--exclude:module')
 677     else:
 678         args.append('--include:module')
 679     if EXCLUDE_LOCAL:
 680         args.append('--exclude:local')
 681     else:
 682         args.append('--include:local')
 683
 684     return ' '.join(args)
 685
 686 def generate_includes(includes):
 687     """Generates the include lines of the pch.
 688     """
 689     lines = []
 690     for osname, group in includes.iteritems():
 691         if not len(group):
 692             continue
 693
 694         if len(osname):
 695             not_eq = ''
 696             if osname[0] == '!':
 697                 not_eq = '!'
 698                 osname = osname[1:]
 699             lines.append('')
 700             lines.append('#if {}defined({})'.format(not_eq, osname))
 701
 702         for i in group:
 703             lines.append(i)
 704
 705         if len(osname):
 706             lines.append('#endif')
 707
 708     return lines
 709
 710 def generate(includes, libname, filename, module):
 711     header = \
 712 """/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 713 /*
 714  * This file is part of the LibreOffice project.
 715  *
 716  * This Source Code Form is subject to the terms of the Mozilla Public
 717  * License, v. 2.0. If a copy of the MPL was not distributed with this
 718  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 719  */
 720
 721 /*
 722  This file has been autogenerated by update_pch.sh. It is possible to edit it
 723  manually (such as when an include file has been moved/renamed/removed). All such
 724  manual changes will be rewritten by the next run of update_pch.sh (which presumably
 725  also fixes all possible problems, so it's usually better to use it).
 726 """
 727
 728     footer = \
 729 """
 730 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
 731 """
 732     import datetime
 733
 734     with open(filename, 'w') as f:
 735         f.write(header)
 736         f.write('\n Generated on {} using:\n {}\n'.format(
 737                 datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
 738                 make_command_line()))
 739         f.write('\n If after updating build fails, use the following command to locate conflicting headers:\n ./bin/update_pch_bisect {} "make {}.build" --find-conflicts\n*/\n'.format(
 740                 filename, module))
 741
 742         # sal needs this for rand_s()
 743         if module == 'sal' and libname == 'sal':
 744             sal_define = """
 745 #if defined(_WIN32)
 746 #if !defined _CRT_RAND_S
 747 #define _CRT_RAND_S
 748 #endif
 749 #endif
 750 """
 751             f.write(sal_define)
 752
 753         # Dump the headers.
 754         f.write('\n')
 755         for i in includes:
 756             f.write(i + '\n')
 757
 758         # Some libraries pull windows headers that aren't self contained.
 759         if (module == 'connectivity' and libname == 'ado') or \
 760            (module == 'xmlsecurity' and libname == 'xsec_xmlsec'):
 761             ado_define = """
 762 // Cleanup windows header macro pollution.
 763 #if defined(_WIN32) && defined(WINAPI)
 764 #   include <postwin.h>
 765 #   undef RGB
 766 #endif
 767 """
 768             f.write(ado_define)
 769
 770         f.write(footer)
 771
 772 def remove_from_tree(filename, tree):
 773     # Remove this file, if top-level.
 774     incs = tree.pop(filename, [])
 775     for i in incs:
 776         tree = remove_from_tree(i, tree)
 777
 778     # Also remove if included from another.
 779     for (k, v) in tree.iteritems():
 780         if filename in v:
 781             v.remove(filename)
 782
 783     return tree
 784
 785 def tree_to_list(includes, filename, tree):
 786     if filename in includes:
 787         return includes
 788     includes.append(filename)
 789     #incs = tree.pop(filename, [])
 790     incs = tree[filename] if filename in tree else []
 791     for i in incs:
 792         tree_to_list(includes, i, tree)
 793
 794     return includes
 795
 796 def promote(includes):
 797     """ Common library headers are heavily
 798         referenced, even if they are included
 799         from a few places.
 800         Here we separate them to promote
 801         their inclusion in the final pch.
 802     """
 803     promo = []
 804     for inc in includes:
 805         if inc.startswith('boost') or \
 806            inc.startswith('sal') or \
 807            inc.startswith('osl') or \
 808            inc.startswith('rtl'):
 809             promo.append(inc)
 810     return promo
 811
 812 def make_pch_filename(root, module, libname):
 813     """ PCH files are stored here:
 814         <root>/<module>/inc/pch/precompiled_<libname>.hxx
 815     """
 816
 817     path = os.path.join(root, module)
 818     path = os.path.join(path, 'inc')
 819     path = os.path.join(path, 'pch')
 820     path = os.path.join(path, 'precompiled_' + libname + '.hxx')
 821     return path
 822
 823 def main():
 824
 825     global CUTOFF
 826     global EXCLUDE_MODULE
 827     global EXCLUDE_LOCAL
 828     global EXCLUDE_SYSTEM
 829     global SILENT
 830
 831     root = '.'
 832     module = sys.argv[1]
 833     libname = sys.argv[2]
 834     header = make_pch_filename(root, module, libname)
 835
 836     if not os.path.exists(os.path.join(root, module)):
 837         raise Exception('Error: module [{}] not found.'.format(module))
 838
 839     key = '{}.{}'.format(module, libname)
 840     if key in DEFAULTS:
 841         # Load the module-specific defaults.
 842         CUTOFF = DEFAULTS[key][0]
 843         EXCLUDE_SYSTEM = DEFAULTS[key][1]
 844         EXCLUDE_MODULE = DEFAULTS[key][2]
 845         EXCLUDE_LOCAL = DEFAULTS[key][3]
 846
 847     force_update = False
 848     for x in xrange(3, len(sys.argv)):
 849         i = sys.argv[x]
 850         if i.startswith('--cutoff='):
 851             CUTOFF = int(i.split('=')[1])
 852         elif i.startswith('--exclude:'):
 853             cat = i.split(':')[1]
 854             if cat == 'module':
 855                 EXCLUDE_MODULE = True
 856             elif cat == 'local':
 857                 EXCLUDE_LOCAL = True
 858             elif cat == 'system':
 859                 EXCLUDE_SYSTEM = True
 860         elif i.startswith('--include:'):
 861             cat = i.split(':')[1]
 862             if cat == 'module':
 863                 EXCLUDE_MODULE = False
 864             elif cat == 'local':
 865                 EXCLUDE_LOCAL = False
 866             elif cat == 'system':
 867                 EXCLUDE_SYSTEM = False
 868         elif i == '--silent':
 869             SILENT = True
 870         elif i == '--force':
 871             force_update = True
 872         else:
 873             sys.stderr.write('Unknown option [{}].'.format(i))
 874             return 1
 875
 876     filter_local = Filter_Local(root, module, \
 877                                 not EXCLUDE_MODULE, \
 878                                 not EXCLUDE_LOCAL)
 879
 880     # Read input.
 881     makefile = 'Library_{}.mk'.format(libname)
 882     groups = process_makefile(root, module, makefile)
 883
 884     generic = []
 885     for osname, group in groups.iteritems():
 886         if not len(group):
 887             continue
 888
 889         includes = []
 890         for filename in group:
 891             includes += process_source(root, module, filename)
 892
 893         # Save unique top-level includes.
 894         unique = set(includes)
 895         promoted = promote(unique)
 896
 897         # Process includes.
 898         includes = remove_rare(includes)
 899         includes = process_list(includes, lambda x: filter_ignore(x, module))
 900         includes = process_list(includes, filter_local.proc)
 901
 902         # Remove the already included ones.
 903         for inc in includes:
 904             unique.discard(inc)
 905
 906         # Explode the excluded ones.
 907         tree = {i:[] for i in includes}
 908         tree = explode(root, module, unique, tree, filter_local, not EXCLUDE_MODULE)
 909
 910         # Remove the already included ones from the tree.
 911         for inc in includes:
 912             filename = get_filename(inc)
 913             tree = remove_from_tree(filename, tree)
 914
 915         extra = []
 916         for (k, v) in tree.iteritems():
 917             extra += tree_to_list([], k, tree)
 918
 919         promoted += promote(extra)
 920         promoted = process_list(promoted, lambda x: filter_ignore(x, module))
 921         promoted = process_list(promoted, filter_local.proc)
 922         promoted = set(promoted)
 923         # If a promoted header includes others, remove the rest.
 924         for (k, v) in tree.iteritems():
 925             if k in promoted:
 926                 for i in v:
 927                     promoted.discard(i)
 928         includes += [x for x in promoted]
 929
 930         extra = remove_rare(extra)
 931         extra = process_list(extra, lambda x: filter_ignore(x, module))
 932         extra = process_list(extra, filter_local.proc)
 933         includes += extra
 934
 935         includes = [x for x in set(includes)]
 936         fixes = fixup(includes, module)
 937         fixes = map(lambda x: '#include <' + x + '>', fixes)
 938
 939         includes = map(lambda x: '#include <' + x + '>', includes)
 940         sorted = sort_by_category(includes, module, filter_local)
 941         includes = fixes + sorted
 942
 943         if len(osname):
 944             for i in generic:
 945                 if i in includes:
 946                     includes.remove(i)
 947
 948         groups[osname] = includes
 949         if not len(osname):
 950             generic = includes
 951
 952     # Open the old pch and compare its contents
 953     # with new includes.
 954     # Clobber only if they are different.
 955     with open(header, 'r') as f:
 956         old_pch_lines = [x.strip() for x in f.readlines()]
 957         new_lines = generate_includes(groups)
 958         # Find the first include in the old pch.
 959         start = -1
 960         for i in xrange(len(old_pch_lines)):
 961             if old_pch_lines[i].startswith('#include'):
 962                 start = i
 963                 break
 964         # Clobber if there is a mismatch.
 965         if force_update or start < 0 or (len(old_pch_lines) - start < len(new_lines)):
 966             generate(new_lines, libname, header, module)
 967             return 0
 968         else:
 969             for i in xrange(len(new_lines)):
 970                 if new_lines[i] != old_pch_lines[start + i]:
 971                     generate(new_lines, libname, header, module)
 972                     return 0
 973             else:
 974                 # Identical, but see if new pch removed anything.
 975                 for i in xrange(start + len(new_lines), len(old_pch_lines)):
 976                     if '#include' in old_pch_lines[i]:
 977                         generate(new_lines, libname, header, module)
 978                         return 0
 979
 980     # Didn't update.
 981     return 1
 982
 983 if __name__ == '__main__':
 984     """ Process all the includes in a Module
 985         to make into a PCH file.
 986         Run without arguments for unittests,
 987         and to see usage.
 988     """
 989
 990     if len(sys.argv) >= 3:
 991         status = main()
 992         sys.exit(status)
 993
 994     print('Usage: {} <Module name> <Library name> [options]'.format(sys.argv[0]))
 995     print('    Always run from the root of LO repository.\n')
 996     print('    Options:')
 997     print('    --cutoff=<count> - Threshold to excluding headers.')
 998     print('    --exclude:<category> - Exclude category-specific headers.')
 999     print('    --include:<category> - Include category-specific headers.')
1000     print('    --force - Force updating the pch even when nothing changes.')
1001     print('    Categories:')
1002     print('         module - Headers in /inc directory of a module.')
1003     print('         local  - Headers local to a source file.')
1004     print('         system - Platform-specific headers.')
1005     print('    --silent - print only errors.')
1006     print('\nRunning unit-tests...')
1007
1008
1009 class TestMethods(unittest.TestCase):
1010
1011     def test_sanitize(self):
1012         self.assertEqual(sanitize('#include "blah/file.cxx"'),
1013                                 '#include <blah/file.cxx>')
1014         self.assertEqual(sanitize('  #include\t"blah/file.cxx" '),
1015                                 '#include <blah/file.cxx>')
1016         self.assertEqual(sanitize('  '),
1017                                 '')
1018
1019     def test_filter_ignore(self):
1020         self.assertEqual(filter_ignore('blah/file.cxx', 'mod'),
1021                                      '')
1022         self.assertEqual(filter_ignore('vector', 'mod'),
1023                                      'vector')
1024         self.assertEqual(filter_ignore('file.cxx', 'mod'),
1025                                      '')
1026
1027     def test_remove_rare(self):
1028         self.assertEqual(remove_rare([]),
1029                                 [])
1030
1031 class TestMakefileParser(unittest.TestCase):
1032
1033     def setUp(self):
1034         global EXCLUDE_SYSTEM
1035         EXCLUDE_SYSTEM = False
1036
1037     def test_parse_singleline_eval(self):
1038         source = "$(eval $(call gb_Library_Library,sal))"
1039         lines = source.split('\n')
1040         groups = {'':[]}
1041         groups = parse_makefile(groups, lines, 0, None, 0)
1042         self.assertEqual(len(groups), 1)
1043         self.assertEqual(len(groups['']), 0)
1044
1045     def test_parse_multiline_eval(self):
1046         source = """$(eval $(call gb_Library_set_include,sal,\\
1047         $$(INCLUDE) \\
1048         -I$(SRCDIR)/sal/inc \\
1049 ))
1050 """
1051         lines = source.split('\n')
1052         groups = {'':[]}
1053         groups = parse_makefile(groups, lines, 0, None, 0)
1054         self.assertEqual(len(groups), 1)
1055         self.assertEqual(len(groups['']), 0)
1056
1057     def test_parse_multiline_eval_with_if(self):
1058         source = """$(eval $(call gb_Library_add_defs,sal,\\
1059         $(if $(filter $(OS),IOS), \\
1060                 -DNO_CHILD_PROCESSES \\
1061         ) \\
1062 ))
1063 """
1064         lines = source.split('\n')
1065         groups = {'':[]}
1066         groups = parse_makefile(groups, lines, 0, None, 0)
1067         self.assertEqual(len(groups), 1)
1068         self.assertEqual(len(groups['']), 0)
1069
1070     def test_parse_multiline_add_with_if(self):
1071         source = """$(eval $(call gb_Library_add_exception_objects,sal,\\
1072         sal/osl/unx/time \\
1073         $(if $(filter DESKTOP,$(BUILD_TYPE)), sal/osl/unx/salinit) \\
1074 ))
1075 """
1076         lines = source.split('\n')
1077         groups = {'':[]}
1078         groups = parse_makefile(groups, lines, 0, None, 0)
1079         self.assertEqual(len(groups), 1)
1080         self.assertEqual(len(groups['']), 1)
1081         self.assertEqual(groups[''][0], 'sal/osl/unx/time.cxx')
1082
1083     def test_parse_if_else(self):
1084         source = """ifeq ($(OS),MACOSX)
1085 $(eval $(call gb_Library_add_exception_objects,sal,\\
1086         sal/osl/mac/mac \\
1087 ))
1088 else
1089 $(eval $(call gb_Library_add_exception_objects,sal,\\
1090         sal/osl/unx/uunxapi \\
1091 ))
1092 endif
1093 """
1094         lines = source.split('\n')
1095         groups = {'':[]}
1096         groups = parse_makefile(groups, lines, 0, None, 0)
1097         self.assertEqual(len(groups), 3)
1098         self.assertEqual(len(groups['']), 0)
1099         self.assertEqual(len(groups['MACOSX']), 1)
1100         self.assertEqual(len(groups['!MACOSX']), 1)
1101         self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx')
1102         self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx')
1103
1104     def test_parse_nested_if(self):
1105         source = """ifeq ($(OS),MACOSX)
1106 $(eval $(call gb_Library_add_exception_objects,sal,\\
1107         sal/osl/mac/mac \\
1108 ))
1109 else
1110 $(eval $(call gb_Library_add_exception_objects,sal,\\
1111         sal/osl/unx/uunxapi \\
1112 ))
1113
1114 ifeq ($(OS),LINUX)
1115 $(eval $(call gb_Library_add_exception_objects,sal,\\
1116         sal/textenc/context \\
1117 ))
1118 endif
1119 endif
1120 """
1121         lines = source.split('\n')
1122         groups = {'':[]}
1123         groups = parse_makefile(groups, lines, 0, None, 0)
1124         self.assertEqual(len(groups), 4)
1125         self.assertEqual(len(groups['']), 0)
1126         self.assertEqual(len(groups['MACOSX']), 1)
1127         self.assertEqual(len(groups['!MACOSX']), 1)
1128         self.assertEqual(len(groups['LINUX']), 1)
1129         self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx')
1130         self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx')
1131         self.assertEqual(groups['LINUX'][0], 'sal/textenc/context.cxx')
1132
1133     def test_parse_exclude_system(self):
1134         source = """ifeq ($(OS),MACOSX)
1135 $(eval $(call gb_Library_add_exception_objects,sal,\\
1136         sal/osl/mac/mac \\
1137 ))
1138 else
1139 $(eval $(call gb_Library_add_exception_objects,sal,\\
1140         sal/osl/unx/uunxapi \\
1141 ))
1142
1143 ifeq ($(OS),LINUX)
1144 $(eval $(call gb_Library_add_exception_objects,sal,\\
1145         sal/textenc/context \\
1146 ))
1147 endif
1148 endif
1149 """
1150         global EXCLUDE_SYSTEM
1151         EXCLUDE_SYSTEM = True
1152
1153         lines = source.split('\n')
1154         groups = {'':[]}
1155         groups = parse_makefile(groups, lines, 0, None, 0)
1156         self.assertEqual(len(groups), 1)
1157         self.assertEqual(len(groups['']), 0)
1158
1159     def test_parse_filter(self):
1160         source = """ifneq ($(filter $(OS),MACOSX IOS),)
1161 $(eval $(call gb_Library_add_exception_objects,sal,\\
1162         sal/osl/unx/osxlocale \\
1163 ))
1164 endif
1165 """
1166         # Filter is still unsupported.
1167         lines = source.split('\n')
1168         groups = {'':[]}
1169         groups = parse_makefile(groups, lines, 0, None, 0)
1170         self.assertEqual(len(groups), 1)
1171         self.assertEqual(len(groups['']), 0)
1172
1173 unittest.main()
1174
1175 # vim: set et sw=4 ts=4 expandtab: