From 0cb87375ad87f3d6cc77e1fdcf0e819bad400b54 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 27 Nov 2024 10:39:12 -0700 Subject: [PATCH] Further tweak CPPDEFINES repleacement in scanner Replacement is now limited to five passes, to avoid going into an endless loop in pathlogical cases where there are three or more macros that circularly refer to each other. No error is reported in this case. Replacement is now only done for the otional C Conditional scanner, not for the classical C scanner. Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 ++++++ RELEASE.txt | 13 +++++++---- SCons/Scanner/C.py | 60 +++++++++++++++++++++++++++++-------------------- SCons/Scanner/CTests.py | 2 +- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6411fcb56..2609355a9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -170,6 +170,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - The C scanner now does (limited) macro replacement on the values in CPPDEFINES, to improve results of conditional source file inclusion (issue #4523). + - The (optional) C Conditional Scanner now does limited macro + replacement on the contents of CPPDEFINES, to improve finding deps + that are conditionally included. Previously replacement was only + done on macro definitions found in the file being scanned. + Only object-like macros are replaced (not function-like), and + only on a whole-word basis; recursion is limited to five levels + and does not error out if that limit is reached (issue #4523). RELEASE 4.8.1 - Tue, 03 Sep 2024 17:22:20 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 93e137f4d..da8f74d3e 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -144,15 +144,20 @@ FIXES not designed to work for the root user. - Make sure unknown variables from a Variables file are recognized - as such (issue #4645) + as such. Previously only unknowns from the command line were + recognized (issue #4645). - Update ninja tool to use ninja.BIN_DIR to find pypi packaged ninja binary. python ninja package version 1.11.1.2 changed the location and previous logic no longer worked. -- The C scanner now does (limited) macro replacement on the values in - CPPDEFINES, to improve results of conditional source file inclusion - (issue #4523). +- The (optional) C Conditional Scanner now does limited macro + replacement on the contents of CPPDEFINES, to improve finding deps + that are conditionally included. Previously replacement was only + done on macro definitions found in the file being scanned. + Only object-like macros are replaced (not function-like), and + only on a whole-word basis; recursion is limited to five levels + and does not error out if that limit is reached (issue #4523). IMPROVEMENTS ------------ diff --git a/SCons/Scanner/C.py b/SCons/Scanner/C.py index ad332b8ad..e111778c9 100644 --- a/SCons/Scanner/C.py +++ b/SCons/Scanner/C.py @@ -29,6 +29,7 @@ add_scanner() for each affected suffix. """ from typing import Dict + import SCons.Node.FS import SCons.cpp import SCons.Util @@ -66,12 +67,12 @@ class SConsCPPScanner(SCons.cpp.PreProcessor): self.missing.append((file, self.current_file)) return '' -def dictify_CPPDEFINES(env) -> dict: +def dictify_CPPDEFINES(env, replace: bool = False) -> dict: """Return CPPDEFINES converted to a dict for preprocessor emulation. The concept is similar to :func:`~SCons.Defaults.processDefines`: turn the values stored in an internal form in ``env['CPPDEFINES']`` - into one usable in a specific context - in this case the cpp-like + into one needed for a specific context - in this case the cpp-like work the C/C++ scanner will do. We can't reuse ``processDefines`` output as that's a list of strings for the command line. We also can't pass the ``CPPDEFINES`` variable directly to the ``dict`` constructor, @@ -79,47 +80,50 @@ def dictify_CPPDEFINES(env) -> dict: after ``Append`` and relatives has been called we know for sure it will be a deque of tuples. - Since the result here won't pass through a real preprocessor, simulate - some of the macro replacement that would take place if it did, or some - conditional inclusions might come out wrong. A bit of an edge case, but - does happen (GH #4623). See 6.10.5 in the C standard and 15.6 in the - C++ standard). + If requested (*replace* is true), simulate some of the macro + replacement that would take place if an actual preprocessor ran, + to avoid some conditional inclusions comeing out wrong. A bit + of an edge case, but does happen (GH #4623). See 6.10.5 in the C + standard and 15.6 in the C++ standard). + + Args: + replace: if true, simulate macro replacement .. versionchanged:: NEXT_RELEASE - Simple macro replacement added. + Simple macro replacement added, and *replace* arg to enable it. """ def _replace(mapping: Dict) -> Dict: """Simplistic macro replacer for dictify_CPPDEFINES. - *mapping* is scanned for any value that is the same as a key in - the dict, and is replaced by the value of that key; the process - is repeated. This is a cheap approximation of the C preprocessor's + Scan *mapping* for a value that is the same as a key in the dict, + and replace with the value of that key; the process is repeated a few + times, but not forever in case someone left a case that can't be + fully resolved. This is a cheap approximation of the preprocessor's macro replacement rules with no smarts - it doesn't "look inside" the values, so only triggers on object-like macros, not on - function-like macros, and will not work on complex values, - e.g. a value like ``(1UL << PR_MTE_TCF_SHIFT)`` would not have - ``PR_MTE_TCF_SHIFT`` replaced if it was also in ``CPPDEFINES``, - but rather left as-is for the scanner to do comparisons against. + function-like macros, and will not work on complex values, e.g. + a value like ``(1UL << PR_MTE_TCF_SHIFT)`` would not have + ``PR_MTE_TCF_SHIFT`` replaced if that was also a key in ``CPPDEFINES``. Args: mapping: a dictionary representing macro names and replacements. Returns: - a dictionary with substitutions made. + a dictionary with replacements made. """ old_ns = mapping - ns = {} - while True: + loops = 0 + while loops < 5: # don't recurse forever in case there's circular data ns = {k: old_ns[v] if v in old_ns else v for k, v in old_ns.items()} if old_ns == ns: break old_ns = ns + loops += 1 return ns cppdefines = env.get('CPPDEFINES', {}) - result = {} - if cppdefines is None: - return result + if not cppdefines: + return {} if SCons.Util.is_Tuple(cppdefines): # single macro defined in a tuple @@ -130,6 +134,7 @@ def dictify_CPPDEFINES(env) -> dict: if SCons.Util.is_Sequence(cppdefines): # multiple (presumably) macro defines in a deque, list, etc. + result = {} for c in cppdefines: if SCons.Util.is_Sequence(c): try: @@ -146,7 +151,9 @@ def dictify_CPPDEFINES(env) -> dict: else: # don't really know what to do here result[c] = None - return _replace(result) + if replace: + return _replace(result) + return(result) if SCons.Util.is_String(cppdefines): # single macro define in a string @@ -158,7 +165,9 @@ def dictify_CPPDEFINES(env) -> dict: if SCons.Util.is_Dict(cppdefines): # already in the desired form - return _replace(result) + if replace: + return _replace(cppdefines) + return cppdefines return {cppdefines: None} @@ -177,7 +186,9 @@ class SConsCPPScannerWrapper: def __call__(self, node, env, path=()): cpp = SConsCPPScanner( - current=node.get_dir(), cpppath=path, dict=dictify_CPPDEFINES(env) + current=node.get_dir(), + cpppath=path, + dict=dictify_CPPDEFINES(env, replace=True), ) result = cpp(node) for included, includer in cpp.missing: @@ -190,6 +201,7 @@ class SConsCPPScannerWrapper: def recurse_nodes(self, nodes): return nodes + def select(self, node): return self diff --git a/SCons/Scanner/CTests.py b/SCons/Scanner/CTests.py index 835593767..b0fdb566e 100644 --- a/SCons/Scanner/CTests.py +++ b/SCons/Scanner/CTests.py @@ -580,7 +580,7 @@ class dictify_CPPDEFINESTestCase(unittest.TestCase): ("RVALUE", "AVALUE"), ] ) - d = SCons.Scanner.C.dictify_CPPDEFINES(env) + d = SCons.Scanner.C.dictify_CPPDEFINES(env, replace=True) expect = {"STRING": "VALUE", "REPLACEABLE": "AVALUE", "RVALUE": "AVALUE"} self.assertEqual(d, expect) -- 2.11.4.GIT