tools/delete_includes.py

   1 #!/usr/bin/python3
   2
   3 # Martin Mathieson
   4 # Look for and removes unnecessary includes in .cpp or .c files
   5 # Run from wireshark source folder as e.g.,
   6 #    ./tools/delete_includes.py --build-folder ~/wireshark-build/ --folder epan/dissectors/
   7 #
   8 # Wireshark - Network traffic analyzer
   9 # By Gerald Combs <gerald@wireshark.org>
  10 # Copyright 1998 Gerald Combs
  11 #
  12 # SPDX-License-Identifier: GPL-2.0-or-later
  13 #
  14
  15 import subprocess
  16 import os
  17 import sys
  18 import shutil
  19 import argparse
  20 import signal
  21 from pathlib import Path
  22
  23
  24 # Try to exit soon after Ctrl-C is pressed.
  25 should_exit = False
  26
  27 def signal_handler(sig, frame):
  28     global should_exit
  29     should_exit = True
  30     print('You pressed Ctrl+C - exiting')
  31
  32 signal.signal(signal.SIGINT, signal_handler)
  33
  34 # For text colouring/highlighting.
  35 class bcolors:
  36     HEADER = '\033[95m'
  37     OKBLUE = '\033[94m'
  38     OKGREEN = '\033[92m'
  39     ADDED = '\033[45m'
  40     WARNING = '\033[93m'
  41     FAIL = '\033[91m'
  42     ENDC = '\033[0m'
  43     BOLD = '\033[1m'
  44     UNDERLINE = '\033[4m'
  45
  46
  47
  48 # command-line args
  49 #
  50 # Controls which dissector files should be checked.  If no args given, will just
  51 # scan whole epan/dissectors folder.
  52 parser = argparse.ArgumentParser(description='Check calls in dissectors')
  53 # required
  54 parser.add_argument('--build-folder', action='store', required=True,
  55                     help='specify individual dissector file to test')
  56 parser.add_argument('--file', action='append',
  57                     help='specify individual dissector file to test')
  58 parser.add_argument('--folder', action='store', default=os.path.join('epan', 'dissectors'),
  59                     help='specify folder to test, relative to current/wireshark folder')
  60 parser.add_argument('--commits', action='store',
  61                     help='last N commits to check')
  62 parser.add_argument('--open', action='store_true',
  63                     help='check open files')
  64 parser.add_argument('--first-file', action='store',
  65                     help='first file in folder to test')
  66 parser.add_argument('--last-file', action='store',
  67                     help='last file in folder to test')
  68 args = parser.parse_args()
  69
  70
  71 test_folder = os.path.join(os.getcwd(), args.folder)
  72
  73
  74 # Usually only building one module, so no -j benefit?
  75 make_command = ['cmake', '--build', args.build_folder]
  76 if sys.platform.startswith('win'):
  77     make_command += ['--config', 'RelWithDebInfo']
  78
  79
  80
  81 # A list of header files that it is not safe to uninclude, as doing so
  82 # has been seen to cause link failures against implemented functions...
  83 # TODO: some of these could probably be removed on more permissive platforms.
  84 includes_to_keep = {
  85     'config.h',
  86     'epan/packet.h',
  87     'stdlib.h',
  88     'math.h',
  89     'errno.h',
  90     'string.h',
  91     'prefs.h',
  92     # These are probably mostly redundant in that they are now covered by the check
  93     # for 'self-includes'...
  94     'x11-keysym.h',
  95     'packet-atm.h',
  96     'packet-atalk.h',
  97     'packet-ppp.h',
  98     'packet-scsi-mmc.h',
  99     'packet-tls.h'
 100 }
 101
 102
 103 # Build stats.
 104 class BuildStats:
 105     def __init__(self):
 106         self.files_examined = 0
 107         self.includes_tested = 0
 108         self.includes_deleted = 0
 109         self.files_not_built_list = []
 110         self.generated_files_ignored = []
 111         self.includes_to_keep_kept = 0
 112
 113     def showSummary(self):
 114         print('\n\n')
 115         print('Summary')
 116         print('=========')
 117         print('files examined:   %d' %  self.files_examined)
 118         print('includes tested:  %d' %  self.includes_tested)
 119         print('includes deleted: %d' %  self.includes_deleted)
 120         print('files not built:  %d' %  len(self.files_not_built_list))
 121         for abandoned_file in self.files_not_built_list:
 122             print('     %s' % abandoned_file)
 123         print('generated files not tested: %d' % len(self.generated_files_ignored))
 124         for generated_file in self.generated_files_ignored:
 125             print('     %s' % generated_file)
 126         print('includes kept as not safe to remove: %d' % self.includes_to_keep_kept)
 127
 128 stats = BuildStats()
 129
 130
 131 # We want to confirm that this file is actually built as part of the build.
 132 # To do this, add some nonsense to the front of the file and confirm that the
 133 # build then fails.  If it doesn't, won't want to remove #includes from that file!
 134 def test_file_is_built(filename):
 135     print('test_file_is_built(', filename, ')')
 136     temp_filename = filename + '.tmp'
 137
 138     f_read = open(filename, 'r')
 139     write_filename = filename + '.new'
 140     f_write = open(write_filename, 'w')
 141     # Write the file with nonsense at start.
 142     f_write.write('NO WAY THIS FILE BUILDS!!!!!')
 143     # Copy remaining lines as-is.
 144     for line in f_read:
 145         f_write.write(line)
 146     f_read.close()
 147     f_write.close()
 148     # Backup file, and do this build with the one we wrote.
 149     shutil.copy(filename, temp_filename)
 150     shutil.copy(write_filename, filename)
 151
 152     # Try the build.
 153     result = subprocess.call(make_command)
 154     # Restore proper file & delete temp files
 155     shutil.copy(temp_filename, filename)
 156     os.remove(temp_filename)
 157     os.remove(write_filename)
 158
 159     if result == 0:
 160         # Build succeeded so this file wasn't in it
 161         return False
 162     else:
 163         # Build failed so this file *is* part of it
 164         return True
 165
 166
 167 # Function to test removal of each #include from a file in turn.
 168 # At the end, only those that appear to be needed will be left.
 169 def test_file(filename):
 170     global stats
 171
 172     print('\n------------------------------')
 173     print(bcolors.OKBLUE, bcolors.BOLD, 'Testing', filename, bcolors.ENDC)
 174
 175     temp_filename = filename + '.tmp'
 176
 177     # Test if file seems to be part of the build.
 178     is_built = test_file_is_built(filename)
 179     if not is_built:
 180         print(bcolors.WARNING, '***** File not used in build, so ignore!!!!', bcolors.ENDC)
 181         # TODO: should os.path.join with root before adding?
 182         stats.files_not_built_list.append(filename)
 183         return
 184     else:
 185         print('This file is part of the build')
 186
 187     # OK, we are going to test removing includes from this file.
 188     tested_line_number = 0
 189
 190     # Don't want to delete 'self-includes', so prepare filename.
 191     module_name = Path(filename).stem
 192     module_header = module_name + '.h'
 193
 194     # Loop around, finding all possible include lines to comment out
 195     while (True):
 196         if should_exit:
 197             exit(1)
 198
 199         have_deleted_line = False
 200         result = 0
 201
 202         # Open read & write files
 203         f_read = open(filename, 'r')
 204         write_filename = filename + '.new'
 205         f_write = open(write_filename, 'w')
 206
 207         # Walk the file again looking for another place to comment out an include
 208         this_line_number = 1
 209         hash_if_level = 0
 210
 211         for line in f_read:
 212             this_line_deleted = False
 213
 214             # Maintain view of how many #if or #ifdefs we are in.
 215             # Don't want to remove any includes that may not be active in this build.
 216             if line.startswith('#if'):
 217                 hash_if_level = hash_if_level + 1
 218
 219             if line.startswith('#endif'):
 220                 if hash_if_level > 1:
 221                     hash_if_level = hash_if_level - 1
 222
 223             # Consider deleting this line have haven't already reached.
 224             if (not have_deleted_line and (tested_line_number < this_line_number)):
 225
 226                 # Test line for starting with #include, and eligible for deletion.
 227                 if line.startswith('#include ') and hash_if_level == 0 and line.find(module_header) == -1:
 228                     # Check that this isn't a header file that known unsafe to uninclude.
 229                     allowed_to_delete = True
 230                     for entry in includes_to_keep:
 231                         if line.find(entry) != -1:
 232                             allowed_to_delete = False
 233                             stats.includes_to_keep_kept += 1
 234                             continue
 235
 236                     if allowed_to_delete:
 237                         # OK, actually doing it.
 238                         have_deleted_line = True
 239                         this_line_deleted = True
 240                         tested_line_number = this_line_number
 241
 242             # Write line to output file, unless this very one was deleted.
 243             if not this_line_deleted:
 244                 f_write.write(line)
 245                 this_line_number = this_line_number + 1
 246
 247         # Close both files.
 248         f_read.close()
 249         f_write.close()
 250
 251         # If we commented out a line, try to build file without it.
 252         if (have_deleted_line):
 253             # Test a build.  0 means success, others are failures.
 254             shutil.copy(filename, temp_filename)
 255             shutil.copy(write_filename, filename)
 256
 257             # Try build
 258             result = subprocess.call(make_command)
 259             if result == 0:
 260                 print(bcolors.OKGREEN +bcolors.BOLD + 'Good build' + bcolors.ENDC)
 261                 # Line was eliminated so decrement line counter
 262                 tested_line_number = tested_line_number - 1
 263                 # Inc successes counter
 264                 stats.includes_deleted += 1
 265                 # Good - promote this version by leaving it here!
 266
 267                 # Occasionally fails so delete this file each time.
 268                 # TODO: this is very particular to dissector target...
 269                 if sys.argv[1] == 'dissectors':
 270                     os.remove(os.path.join(args.build_folder, 'vc100.pdb'))
 271             else:
 272                 print(bcolors.FAIL +bcolors.BOLD + 'Bad build' + bcolors.ENDC)
 273                 # Never mind, go back to previous building version
 274                 shutil.copy(temp_filename, filename)
 275
 276             # Inc counter of tried
 277             stats.includes_tested += 1
 278
 279         else:
 280             # Reached the end of the file without making changes, so nothing doing.
 281             # Delete temporary files
 282             if os.path.isfile(temp_filename):
 283                 os.remove(temp_filename)
 284             if os.path.isfile(write_filename):
 285                 os.remove(write_filename)
 286             return
 287
 288 # Test for whether a the given file is under source control
 289 def under_version_control(filename):
 290     # TODO: git command to see if under version control. Check retcode of 'git log <filename>' ?
 291     return True
 292
 293 # Test for whether the given file was automatically generated.
 294 def generated_file(filename):
 295     # Special known case.
 296     if filename == 'register.c':
 297         return True
 298
 299     # Open file
 300     f_read = open(filename, 'r')
 301     lines_tested = 0
 302     for line in f_read:
 303         # The comment to say that its generated is near the top, so give up once
 304         # get a few lines down.
 305         if lines_tested > 10:
 306             f_read.close()
 307             return False
 308         if (line.find('Generated automatically') != -1 or
 309             line.find('Generated Automatically') != -1 or
 310             line.find('Autogenerated from') != -1 or
 311             line.find('is autogenerated') != -1 or
 312             line.find('automatically generated by Pidl') != -1 or
 313             line.find('Created by: The Qt Meta Object Compiler') != -1 or
 314             line.find('This file was generated') != -1 or
 315             line.find('This filter was automatically generated') != -1 or
 316             line.find('This file is auto generated, do not edit!') != -1):
 317
 318             f_read.close()
 319             return True
 320         lines_tested = lines_tested + 1
 321
 322     # OK, looks like a hand-written file!
 323     f_read.close()
 324     return False
 325
 326 def isBuildableFile(filename):
 327     return filename.endswith('.c') or filename.endswith('.cpp')
 328
 329
 330 def findFilesInFolder(folder, recursive=False):
 331     dissector_files = []
 332
 333     if recursive:
 334         for root, subfolders, files in os.walk(folder):
 335             for f in files:
 336                 if should_exit:
 337                     return
 338                 f = os.path.join(root, f)
 339                 dissector_files.append(f)
 340     else:
 341         for f in sorted(os.listdir(folder)):
 342             if should_exit:
 343                 return
 344             filename = os.path.join(folder, f)
 345             dissector_files.append(filename)
 346
 347     return [x for x in filter(isBuildableFile, dissector_files)]
 348
 349
 350 ######################################################################################
 351 # MAIN PROGRAM STARTS HERE
 352 ######################################################################################
 353
 354 # Work out which files we want to look at.
 355 files = []
 356 if args.file:
 357     # Add specified file(s)
 358     for f in args.file:
 359         if not os.path.isfile(f):
 360             print('Chosen file', f, 'does not exist.')
 361             exit(1)
 362         else:
 363             files.append(f)
 364 elif args.folder:
 365     # Add all files from a given folder.
 366     folder = args.folder
 367     if not os.path.isdir(folder):
 368         print('Folder', folder, 'not found!')
 369         exit(1)
 370     # Find files from folder.
 371     print('Looking for files in', folder)
 372     files = findFilesInFolder(folder, recursive=False)
 373
 374
 375 # If first-file/last-file are given, will need to trim files accordingly
 376 if args.first_file:
 377     idx = files.index(args.first_file)
 378     if idx == -1:
 379         print('first-file entry', args.first_file, 'not in list of files to be checked')
 380         exit(1)
 381     else:
 382         files = files[idx:]
 383
 384 if args.last_file:
 385     idx = files.index(args.last_file)
 386     if idx == -1:
 387         print('last-file entry', args.last_file, 'not in list of files to be checked')
 388         exit(1)
 389     else:
 390         files = files[:idx+1]
 391
 392
 393 # Confirm that the build is currently passing, if not give up now.
 394 print(bcolors.OKBLUE,bcolors.BOLD,
 395       'Doing an initial build to check we have a stable base.',
 396       bcolors.ENDC)
 397 result = subprocess.call(make_command)
 398 if result != 0:
 399     print(bcolors.FAIL, bcolors.BOLD, 'Initial build failed - give up now!!!!', bcolors.ENDC)
 400     exit (-1)
 401
 402
 403
 404 # Test each file.
 405 for filename in files:
 406
 407     # Want to filter out generated files that are not checked in.
 408     if not generated_file(filename) and under_version_control(filename):
 409         # OK, try this file
 410         test_file(filename)
 411
 412         # Inc counter
 413         stats.files_examined += 1
 414     else:
 415         if generated_file(filename):
 416             reason = 'generated file...'
 417         if not under_version_control(filename):
 418             reason = 'not under source control'
 419         print('Ignoring %s: %s' % (filename, reason))
 420
 421
 422
 423 # Show summary stats of run
 424 stats.showSummary()