SCons/Tool/MSCommon/common.py

   1 # MIT License
   2 #
   3 # Copyright The SCons Foundation
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining
   6 # a copy of this software and associated documentation files (the
   7 # "Software"), to deal in the Software without restriction, including
   8 # without limitation the rights to use, copy, modify, merge, publish,
   9 # distribute, sublicense, and/or sell copies of the Software, and to
  10 # permit persons to whom the Software is furnished to do so, subject to
  11 # the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  17 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  18 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  19 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  20 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  21 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  22 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23
  24 """
  25 Common helper functions for working with the Microsoft tool chain.
  26 """
  27
  28 import copy
  29 import json
  30 import os
  31 import re
  32 import subprocess
  33 import sys
  34 from contextlib import suppress
  35 from pathlib import Path
  36
  37 import SCons.Util
  38 import SCons.Warnings
  39
  40 class MSVCCacheInvalidWarning(SCons.Warnings.WarningOnByDefault):
  41     pass
  42
  43 # SCONS_MSCOMMON_DEBUG is internal-use so undocumented:
  44 # set to '-' to print to console, else set to filename to log to
  45 LOGFILE = os.environ.get('SCONS_MSCOMMON_DEBUG')
  46 if LOGFILE:
  47     import logging
  48
  49     modulelist = (
  50         # root module and parent/root module
  51         'MSCommon', 'Tool',
  52         # python library and below: correct iff scons does not have a lib folder
  53         'lib',
  54         # scons modules
  55         'SCons', 'test', 'scons'
  56     )
  57
  58     def get_relative_filename(filename, module_list):
  59         if not filename:
  60             return filename
  61         for module in module_list:
  62             try:
  63                 ind = filename.rindex(module)
  64                 return filename[ind:]
  65             except ValueError:
  66                 pass
  67         return filename
  68
  69     class _Debug_Filter(logging.Filter):
  70         # custom filter for module relative filename
  71         def filter(self, record) -> bool:
  72             relfilename = get_relative_filename(record.pathname, modulelist)
  73             relfilename = relfilename.replace('\\', '/')
  74             record.relfilename = relfilename
  75             return True
  76
  77     # Log format looks like:
  78     #   00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3'        [file]
  79     #   debug: 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [stdout]
  80     log_format=(
  81         '%(relativeCreated)05dms'
  82         ':%(relfilename)s'
  83         ':%(funcName)s'
  84         '#%(lineno)s'
  85         ': %(message)s'
  86     )
  87     if LOGFILE == '-':
  88         log_format = 'debug: ' + log_format
  89         log_handler = logging.StreamHandler(sys.stdout)
  90     else:
  91         log_handler = logging.FileHandler(filename=LOGFILE)
  92     log_formatter = logging.Formatter(log_format)
  93     log_handler.setFormatter(log_formatter)
  94     logger = logging.getLogger(name=__name__)
  95     logger.setLevel(level=logging.DEBUG)
  96     logger.addHandler(log_handler)
  97     logger.addFilter(_Debug_Filter())
  98     debug = logger.debug
  99 else:
 100     def debug(x, *args):
 101         return None
 102
 103
 104 # SCONS_CACHE_MSVC_CONFIG is public, and is documented.
 105 CONFIG_CACHE = os.environ.get('SCONS_CACHE_MSVC_CONFIG')
 106 if CONFIG_CACHE in ('1', 'true', 'True'):
 107     CONFIG_CACHE = os.path.join(os.path.expanduser('~'), 'scons_msvc_cache.json')
 108
 109 # SCONS_CACHE_MSVC_FORCE_DEFAULTS is internal-use so undocumented.
 110 CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS = False
 111 if CONFIG_CACHE:
 112     if os.environ.get('SCONS_CACHE_MSVC_FORCE_DEFAULTS') in ('1', 'true', 'True'):
 113         CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS = True
 114
 115 def read_script_env_cache():
 116     """ fetch cached msvc env vars if requested, else return empty dict """
 117     envcache = {}
 118     if CONFIG_CACHE:
 119         try:
 120             p = Path(CONFIG_CACHE)
 121             with p.open('r') as f:
 122                 # Convert the list of cache entry dictionaries read from
 123                 # json to the cache dictionary. Reconstruct the cache key
 124                 # tuple from the key list written to json.
 125                 envcache_list = json.load(f)
 126                 if isinstance(envcache_list, list):
 127                     envcache = {tuple(d['key']): d['data'] for d in envcache_list}
 128                 else:
 129                     # don't fail if incompatible format, just proceed without it
 130                     warn_msg = "Incompatible format for msvc cache file {}: file may be overwritten.".format(
 131                         repr(CONFIG_CACHE)
 132                     )
 133                     SCons.Warnings.warn(MSVCCacheInvalidWarning, warn_msg)
 134                     debug(warn_msg)
 135         except FileNotFoundError:
 136             # don't fail if no cache file, just proceed without it
 137             pass
 138     return envcache
 139
 140
 141 def write_script_env_cache(cache) -> None:
 142     """ write out cache of msvc env vars if requested """
 143     if CONFIG_CACHE:
 144         try:
 145             p = Path(CONFIG_CACHE)
 146             with p.open('w') as f:
 147                 # Convert the cache dictionary to a list of cache entry
 148                 # dictionaries. The cache key is converted from a tuple to
 149                 # a list for compatibility with json.
 150                 envcache_list = [{'key': list(key), 'data': data} for key, data in cache.items()]
 151                 json.dump(envcache_list, f, indent=2)
 152         except TypeError:
 153             # data can't serialize to json, don't leave partial file
 154             with suppress(FileNotFoundError):
 155                 p.unlink()
 156         except IOError:
 157             # can't write the file, just skip
 158             pass
 159
 160
 161 _is_win64 = None
 162
 163
 164 def is_win64() -> bool:
 165     """Return true if running on windows 64 bits.
 166
 167     Works whether python itself runs in 64 bits or 32 bits."""
 168     # Unfortunately, python does not provide a useful way to determine
 169     # if the underlying Windows OS is 32-bit or 64-bit.  Worse, whether
 170     # the Python itself is 32-bit or 64-bit affects what it returns,
 171     # so nothing in sys.* or os.* help.
 172
 173     # Apparently the best solution is to use env vars that Windows
 174     # sets.  If PROCESSOR_ARCHITECTURE is not x86, then the python
 175     # process is running in 64 bit mode (on a 64-bit OS, 64-bit
 176     # hardware, obviously).
 177     # If this python is 32-bit but the OS is 64, Windows will set
 178     # ProgramW6432 and PROCESSOR_ARCHITEW6432 to non-null.
 179     # (Checking for HKLM\Software\Wow6432Node in the registry doesn't
 180     # work, because some 32-bit installers create it.)
 181     global _is_win64
 182     if _is_win64 is None:
 183         # I structured these tests to make it easy to add new ones or
 184         # add exceptions in the future, because this is a bit fragile.
 185         _is_win64 = False
 186         if os.environ.get('PROCESSOR_ARCHITECTURE', 'x86') != 'x86':
 187             _is_win64 = True
 188         if os.environ.get('PROCESSOR_ARCHITEW6432'):
 189             _is_win64 = True
 190         if os.environ.get('ProgramW6432'):
 191             _is_win64 = True
 192     return _is_win64
 193
 194
 195 def read_reg(value, hkroot=SCons.Util.HKEY_LOCAL_MACHINE):
 196     return SCons.Util.RegGetValue(hkroot, value)[0]
 197
 198
 199 def has_reg(value) -> bool:
 200     """Return True if the given key exists in HKEY_LOCAL_MACHINE."""
 201     try:
 202         SCons.Util.RegOpenKeyEx(SCons.Util.HKEY_LOCAL_MACHINE, value)
 203         ret = True
 204     except OSError:
 205         ret = False
 206     return ret
 207
 208 # Functions for fetching environment variable settings from batch files.
 209
 210
 211 def normalize_env(env, keys, force: bool=False):
 212     """Given a dictionary representing a shell environment, add the variables
 213     from os.environ needed for the processing of .bat files; the keys are
 214     controlled by the keys argument.
 215
 216     It also makes sure the environment values are correctly encoded.
 217
 218     If force=True, then all of the key values that exist are copied
 219     into the returned dictionary.  If force=false, values are only
 220     copied if the key does not already exist in the copied dictionary.
 221
 222     Note: the environment is copied."""
 223     normenv = {}
 224     if env:
 225         for k, v in env.items():
 226             normenv[k] = copy.deepcopy(v)
 227
 228         for k in keys:
 229             if k in os.environ and (force or k not in normenv):
 230                 normenv[k] = os.environ[k]
 231
 232     # add some things to PATH to prevent problems:
 233     # Shouldn't be necessary to add system32, since the default environment
 234     # should include it, but keep this here to be safe (needed for reg.exe)
 235     sys32_dir = os.path.join(
 236         os.environ.get("SystemRoot", os.environ.get("windir", r"C:\Windows")), "System32"
 237     )
 238     if sys32_dir not in normenv["PATH"]:
 239         normenv["PATH"] = normenv["PATH"] + os.pathsep + sys32_dir
 240
 241     # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized"
 242     # error starting with Visual Studio 2017, although the script still
 243     # seems to work anyway.
 244     sys32_wbem_dir = os.path.join(sys32_dir, 'Wbem')
 245     if sys32_wbem_dir not in normenv['PATH']:
 246         normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir
 247
 248     # Without Powershell in PATH, an internal call to a telemetry
 249     # function (starting with a VS2019 update) can fail
 250     # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this.
 251     sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\v1.0')
 252     if sys32_ps_dir not in normenv['PATH']:
 253         normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir
 254
 255     debug("PATH: %s", normenv['PATH'])
 256     return normenv
 257
 258
 259 def get_output(vcbat, args=None, env=None):
 260     """Parse the output of given bat file, with given args."""
 261
 262     if env is None:
 263         # Create a blank environment, for use in launching the tools
 264         env = SCons.Environment.Environment(tools=[])
 265
 266     # TODO:  Hard-coded list of the variables that (may) need to be
 267     # imported from os.environ[] for the chain of development batch
 268     # files to execute correctly. One call to vcvars*.bat may
 269     # end up running a dozen or more scripts, changes not only with
 270     # each release but with what is installed at the time. We think
 271     # in modern installations most are set along the way and don't
 272     # need to be picked from the env, but include these for safety's sake.
 273     # Any VSCMD variables definitely are picked from the env and
 274     # control execution in interesting ways.
 275     # Note these really should be unified - either controlled by vs.py,
 276     # or synced with the the common_tools_var # settings in vs.py.
 277     vs_vc_vars = [
 278         'COMSPEC',  # path to "shell"
 279         'OS', # name of OS family: Windows_NT or undefined (95/98/ME)
 280         'VS170COMNTOOLS',  # path to common tools for given version
 281         'VS160COMNTOOLS',
 282         'VS150COMNTOOLS',
 283         'VS140COMNTOOLS',
 284         'VS120COMNTOOLS',
 285         'VS110COMNTOOLS',
 286         'VS100COMNTOOLS',
 287         'VS90COMNTOOLS',
 288         'VS80COMNTOOLS',
 289         'VS71COMNTOOLS',
 290         'VSCOMNTOOLS',
 291         'MSDevDir',
 292         'VSCMD_DEBUG',   # enable logging and other debug aids
 293         'VSCMD_SKIP_SENDTELEMETRY',
 294         'windir', # windows directory (SystemRoot not available in 95/98/ME)
 295     ]
 296     env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False)
 297
 298     if args:
 299         debug("Calling '%s %s'", vcbat, args)
 300         popen = SCons.Action._subproc(env,
 301                                       '"%s" %s & set' % (vcbat, args),
 302                                       stdin='devnull',
 303                                       stdout=subprocess.PIPE,
 304                                       stderr=subprocess.PIPE)
 305     else:
 306         debug("Calling '%s'", vcbat)
 307         popen = SCons.Action._subproc(env,
 308                                       '"%s" & set' % vcbat,
 309                                       stdin='devnull',
 310                                       stdout=subprocess.PIPE,
 311                                       stderr=subprocess.PIPE)
 312
 313     # Use the .stdout and .stderr attributes directly because the
 314     # .communicate() method uses the threading module on Windows
 315     # and won't work under Pythons not built with threading.
 316     with popen.stdout:
 317         stdout = popen.stdout.read()
 318     with popen.stderr:
 319         stderr = popen.stderr.read()
 320
 321     # Extra debug logic, uncomment if necessary
 322     # debug('stdout:%s', stdout)
 323     # debug('stderr:%s', stderr)
 324
 325     # Ongoing problems getting non-corrupted text led to this
 326     # changing to "oem" from "mbcs" - the scripts run presumably
 327     # attached to a console, so some particular rules apply.
 328     # Unfortunately, "oem" not defined in Python 3.5, so get another way
 329     if sys.version_info.major == 3 and sys.version_info.minor < 6:
 330         from ctypes import windll
 331
 332         OEM = "cp{}".format(windll.kernel32.GetConsoleOutputCP())
 333     else:
 334         OEM = "oem"
 335     if stderr:
 336         # TODO: find something better to do with stderr;
 337         # this at least prevents errors from getting swallowed.
 338         sys.stderr.write(stderr.decode(OEM))
 339     if popen.wait() != 0:
 340         raise IOError(stderr.decode(OEM))
 341
 342     return stdout.decode(OEM)
 343
 344
 345 KEEPLIST = (
 346     "INCLUDE",
 347     "LIB",
 348     "LIBPATH",
 349     "PATH",
 350     "VSCMD_ARG_app_plat",
 351     "VCINSTALLDIR",  # needed by clang -VS 2017 and newer
 352     "VCToolsInstallDir",  # needed by clang - VS 2015 and older
 353 )
 354
 355
 356 def parse_output(output, keep=KEEPLIST):
 357     """
 358     Parse output from running visual c++/studios vcvarsall.bat and running set
 359     To capture the values listed in keep
 360     """
 361
 362     # dkeep is a dict associating key: path_list, where key is one item from
 363     # keep, and path_list the associated list of paths
 364     dkeep = {i: [] for i in keep}
 365
 366     # rdk will  keep the regex to match the .bat file output line starts
 367     rdk = {}
 368     for i in keep:
 369         rdk[i] = re.compile('%s=(.*)' % i, re.I)
 370
 371     def add_env(rmatch, key, dkeep=dkeep) -> None:
 372         path_list = rmatch.group(1).split(os.pathsep)
 373         for path in path_list:
 374             # Do not add empty paths (when a var ends with ;)
 375             if path:
 376                 # XXX: For some reason, VC98 .bat file adds "" around the PATH
 377                 # values, and it screws up the environment later, so we strip
 378                 # it.
 379                 path = path.strip('"')
 380                 dkeep[key].append(str(path))
 381
 382     for line in output.splitlines():
 383         for k, value in rdk.items():
 384             match = value.match(line)
 385             if match:
 386                 add_env(match, k)
 387
 388     return dkeep
 389
 390 def get_pch_node(env, target, source):
 391     """
 392     Get the actual PCH file node
 393     """
 394     pch_subst = env.get('PCH', False) and env.subst('$PCH',target=target, source=source, conv=lambda x:x)
 395
 396     if not pch_subst:
 397         return ""
 398
 399     if SCons.Util.is_String(pch_subst):
 400         pch_subst = target[0].dir.File(pch_subst)
 401
 402     return pch_subst
 403
 404
 405 # Local Variables:
 406 # tab-width:4
 407 # indent-tabs-mode:nil
 408 # End:
 409 # vim: set expandtab tabstop=4 shiftwidth=4: