Merge tag 'block-5.11-2021-01-10' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / scripts / clang-tools / gen_compile_commands.py
blob19963708bcf87712384eaedbba359e278cb78321
1 #!/usr/bin/env python
2 # SPDX-License-Identifier: GPL-2.0
4 # Copyright (C) Google LLC, 2018
6 # Author: Tom Roeder <tmroeder@google.com>
8 """A tool for generating compile_commands.json in the Linux kernel."""
10 import argparse
11 import json
12 import logging
13 import os
14 import re
15 import subprocess
17 _DEFAULT_OUTPUT = 'compile_commands.json'
18 _DEFAULT_LOG_LEVEL = 'WARNING'
20 _FILENAME_PATTERN = r'^\..*\.cmd$'
21 _LINE_PATTERN = r'^cmd_[^ ]*\.o := (.* )([^ ]*\.c)$'
22 _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
25 def parse_arguments():
26 """Sets up and parses command-line arguments.
28 Returns:
29 log_level: A logging level to filter log output.
30 directory: The work directory where the objects were built.
31 ar: Command used for parsing .a archives.
32 output: Where to write the compile-commands JSON file.
33 paths: The list of files/directories to handle to find .cmd files.
34 """
35 usage = 'Creates a compile_commands.json database from kernel .cmd files'
36 parser = argparse.ArgumentParser(description=usage)
38 directory_help = ('specify the output directory used for the kernel build '
39 '(defaults to the working directory)')
40 parser.add_argument('-d', '--directory', type=str, default='.',
41 help=directory_help)
43 output_help = ('path to the output command database (defaults to ' +
44 _DEFAULT_OUTPUT + ')')
45 parser.add_argument('-o', '--output', type=str, default=_DEFAULT_OUTPUT,
46 help=output_help)
48 log_level_help = ('the level of log messages to produce (defaults to ' +
49 _DEFAULT_LOG_LEVEL + ')')
50 parser.add_argument('--log_level', choices=_VALID_LOG_LEVELS,
51 default=_DEFAULT_LOG_LEVEL, help=log_level_help)
53 ar_help = 'command used for parsing .a archives'
54 parser.add_argument('-a', '--ar', type=str, default='llvm-ar', help=ar_help)
56 paths_help = ('directories to search or files to parse '
57 '(files should be *.o, *.a, or modules.order). '
58 'If nothing is specified, the current directory is searched')
59 parser.add_argument('paths', type=str, nargs='*', help=paths_help)
61 args = parser.parse_args()
63 return (args.log_level,
64 os.path.abspath(args.directory),
65 args.output,
66 args.ar,
67 args.paths if len(args.paths) > 0 else [args.directory])
70 def cmdfiles_in_dir(directory):
71 """Generate the iterator of .cmd files found under the directory.
73 Walk under the given directory, and yield every .cmd file found.
75 Args:
76 directory: The directory to search for .cmd files.
78 Yields:
79 The path to a .cmd file.
80 """
82 filename_matcher = re.compile(_FILENAME_PATTERN)
84 for dirpath, _, filenames in os.walk(directory):
85 for filename in filenames:
86 if filename_matcher.match(filename):
87 yield os.path.join(dirpath, filename)
90 def to_cmdfile(path):
91 """Return the path of .cmd file used for the given build artifact
93 Args:
94 Path: file path
96 Returns:
97 The path to .cmd file
98 """
99 dir, base = os.path.split(path)
100 return os.path.join(dir, '.' + base + '.cmd')
103 def cmdfiles_for_o(obj):
104 """Generate the iterator of .cmd files associated with the object
106 Yield the .cmd file used to build the given object
108 Args:
109 obj: The object path
111 Yields:
112 The path to .cmd file
114 yield to_cmdfile(obj)
117 def cmdfiles_for_a(archive, ar):
118 """Generate the iterator of .cmd files associated with the archive.
120 Parse the given archive, and yield every .cmd file used to build it.
122 Args:
123 archive: The archive to parse
125 Yields:
126 The path to every .cmd file found
128 for obj in subprocess.check_output([ar, '-t', archive]).decode().split():
129 yield to_cmdfile(obj)
132 def cmdfiles_for_modorder(modorder):
133 """Generate the iterator of .cmd files associated with the modules.order.
135 Parse the given modules.order, and yield every .cmd file used to build the
136 contained modules.
138 Args:
139 modorder: The modules.order file to parse
141 Yields:
142 The path to every .cmd file found
144 with open(modorder) as f:
145 for line in f:
146 ko = line.rstrip()
147 base, ext = os.path.splitext(ko)
148 if ext != '.ko':
149 sys.exit('{}: module path must end with .ko'.format(ko))
150 mod = base + '.mod'
151 # The first line of *.mod lists the objects that compose the module.
152 with open(mod) as m:
153 for obj in m.readline().split():
154 yield to_cmdfile(obj)
157 def process_line(root_directory, command_prefix, file_path):
158 """Extracts information from a .cmd line and creates an entry from it.
160 Args:
161 root_directory: The directory that was searched for .cmd files. Usually
162 used directly in the "directory" entry in compile_commands.json.
163 command_prefix: The extracted command line, up to the last element.
164 file_path: The .c file from the end of the extracted command.
165 Usually relative to root_directory, but sometimes absolute.
167 Returns:
168 An entry to append to compile_commands.
170 Raises:
171 ValueError: Could not find the extracted file based on file_path and
172 root_directory or file_directory.
174 # The .cmd files are intended to be included directly by Make, so they
175 # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
176 # kernel version). The compile_commands.json file is not interepreted
177 # by Make, so this code replaces the escaped version with '#'.
178 prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
180 # Use os.path.abspath() to normalize the path resolving '.' and '..' .
181 abs_path = os.path.abspath(os.path.join(root_directory, file_path))
182 if not os.path.exists(abs_path):
183 raise ValueError('File %s not found' % abs_path)
184 return {
185 'directory': root_directory,
186 'file': abs_path,
187 'command': prefix + file_path,
191 def main():
192 """Walks through the directory and finds and parses .cmd files."""
193 log_level, directory, output, ar, paths = parse_arguments()
195 level = getattr(logging, log_level)
196 logging.basicConfig(format='%(levelname)s: %(message)s', level=level)
198 line_matcher = re.compile(_LINE_PATTERN)
200 compile_commands = []
202 for path in paths:
203 # If 'path' is a directory, handle all .cmd files under it.
204 # Otherwise, handle .cmd files associated with the file.
205 # Most of built-in objects are linked via archives (built-in.a or lib.a)
206 # but some objects are linked to vmlinux directly.
207 # Modules are listed in modules.order.
208 if os.path.isdir(path):
209 cmdfiles = cmdfiles_in_dir(path)
210 elif path.endswith('.o'):
211 cmdfiles = cmdfiles_for_o(path)
212 elif path.endswith('.a'):
213 cmdfiles = cmdfiles_for_a(path, ar)
214 elif path.endswith('modules.order'):
215 cmdfiles = cmdfiles_for_modorder(path)
216 else:
217 sys.exit('{}: unknown file type'.format(path))
219 for cmdfile in cmdfiles:
220 with open(cmdfile, 'rt') as f:
221 result = line_matcher.match(f.readline())
222 if result:
223 try:
224 entry = process_line(directory, result.group(1),
225 result.group(2))
226 compile_commands.append(entry)
227 except ValueError as err:
228 logging.info('Could not add line from %s: %s',
229 cmdfile, err)
231 with open(output, 'wt') as f:
232 json.dump(compile_commands, f, indent=2, sort_keys=True)
235 if __name__ == '__main__':
236 main()