clang/utils/analyzer/SATest.py

   1 #!/usr/bin/env python
   2
   3 import argparse
   4 import sys
   5 import os
   6
   7 from subprocess import call
   8
   9 SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
  10 PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
  11 DEFAULT_LLVM_DIR = os.path.realpath(
  12     os.path.join(SCRIPTS_DIR, os.path.pardir, os.path.pardir, os.path.pardir)
  13 )
  14
  15
  16 def add(parser, args):
  17     import SATestAdd
  18     from ProjectMap import ProjectInfo
  19
  20     if args.source == "git" and (args.origin == "" or args.commit == ""):
  21         parser.error("Please provide both --origin and --commit if source is 'git'")
  22
  23     if args.source != "git" and (args.origin != "" or args.commit != ""):
  24         parser.error(
  25             "Options --origin and --commit don't make sense when " "source is not 'git'"
  26         )
  27
  28     project = ProjectInfo(
  29         args.name[0], args.mode, args.source, args.origin, args.commit
  30     )
  31
  32     SATestAdd.add_new_project(project)
  33
  34
  35 def build(parser, args):
  36     import SATestBuild
  37
  38     SATestBuild.VERBOSE = args.verbose
  39
  40     projects = get_projects(parser, args)
  41     tester = SATestBuild.RegressionTester(
  42         args.jobs,
  43         projects,
  44         args.override_compiler,
  45         args.extra_analyzer_config,
  46         args.extra_checkers,
  47         args.regenerate,
  48         args.strictness,
  49     )
  50     tests_passed = tester.test_all()
  51
  52     if not tests_passed:
  53         sys.stderr.write("ERROR: Tests failed.\n")
  54         sys.exit(42)
  55
  56
  57 def compare(parser, args):
  58     import CmpRuns
  59
  60     choices = [
  61         CmpRuns.HistogramType.RELATIVE.value,
  62         CmpRuns.HistogramType.LOG_RELATIVE.value,
  63         CmpRuns.HistogramType.ABSOLUTE.value,
  64     ]
  65
  66     if args.histogram is not None and args.histogram not in choices:
  67         parser.error(
  68             "Incorrect histogram type, available choices are {}".format(choices)
  69         )
  70
  71     dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
  72     dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
  73
  74     CmpRuns.dump_scan_build_results_diff(
  75         dir_old,
  76         dir_new,
  77         show_stats=args.show_stats,
  78         stats_only=args.stats_only,
  79         histogram=args.histogram,
  80         verbose_log=args.verbose_log,
  81     )
  82
  83
  84 def update(parser, args):
  85     import SATestUpdateDiffs
  86     from ProjectMap import ProjectMap
  87
  88     project_map = ProjectMap()
  89     for project in project_map.projects:
  90         SATestUpdateDiffs.update_reference_results(project, args.git)
  91
  92
  93 def benchmark(parser, args):
  94     from SATestBenchmark import Benchmark
  95
  96     projects = get_projects(parser, args)
  97     benchmark = Benchmark(projects, args.iterations, args.output)
  98     benchmark.run()
  99
 100
 101 def benchmark_compare(parser, args):
 102     import SATestBenchmark
 103
 104     SATestBenchmark.compare(args.old, args.new, args.output)
 105
 106
 107 def get_projects(parser, args):
 108     from ProjectMap import ProjectMap, Size
 109
 110     project_map = ProjectMap()
 111     projects = project_map.projects
 112
 113     def filter_projects(projects, predicate, force=False):
 114         return [
 115             project.with_fields(
 116                 enabled=(force or project.enabled) and predicate(project)
 117             )
 118             for project in projects
 119         ]
 120
 121     if args.projects:
 122         projects_arg = args.projects.split(",")
 123         available_projects = [project.name for project in projects]
 124
 125         # validate that given projects are present in the project map file
 126         for manual_project in projects_arg:
 127             if manual_project not in available_projects:
 128                 parser.error(
 129                     "Project '{project}' is not found in "
 130                     "the project map file. Available projects are "
 131                     "{all}.".format(project=manual_project, all=available_projects)
 132                 )
 133
 134         projects = filter_projects(
 135             projects, lambda project: project.name in projects_arg, force=True
 136         )
 137
 138     try:
 139         max_size = Size.from_str(args.max_size)
 140     except ValueError as e:
 141         parser.error("{}".format(e))
 142
 143     projects = filter_projects(projects, lambda project: project.size <= max_size)
 144
 145     return projects
 146
 147
 148 def docker(parser, args):
 149     if len(args.rest) > 0:
 150         if args.rest[0] != "--":
 151             parser.error("REST arguments should start with '--'")
 152         args.rest = args.rest[1:]
 153
 154     if args.build_image:
 155         docker_build_image()
 156     elif args.shell:
 157         docker_shell(args)
 158     else:
 159         sys.exit(docker_run(args, " ".join(args.rest)))
 160
 161
 162 def docker_build_image():
 163     sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), shell=True))
 164
 165
 166 def docker_shell(args):
 167     try:
 168         # First we need to start the docker container in a waiting mode,
 169         # so it doesn't do anything, but most importantly keeps working
 170         # while the shell session is in progress.
 171         docker_run(args, "--wait", "--detach")
 172         # Since the docker container is running, we can actually connect to it
 173         call("docker exec -it satest bash", shell=True)
 174
 175     except KeyboardInterrupt:
 176         pass
 177
 178     finally:
 179         docker_cleanup()
 180
 181
 182 def docker_run(args, command, docker_args=""):
 183     try:
 184         return call(
 185             "docker run --rm --name satest "
 186             "-v {llvm}:/llvm-project "
 187             "-v {build}:/build "
 188             "-v {clang}:/analyzer "
 189             "-v {scripts}:/scripts "
 190             "-v {projects}:/projects "
 191             "{docker_args} "
 192             "satest-image:latest {command}".format(
 193                 llvm=args.llvm_project_dir,
 194                 build=args.build_dir,
 195                 clang=args.clang_dir,
 196                 scripts=SCRIPTS_DIR,
 197                 projects=PROJECTS_DIR,
 198                 docker_args=docker_args,
 199                 command=command,
 200             ),
 201             shell=True,
 202         )
 203
 204     except KeyboardInterrupt:
 205         docker_cleanup()
 206
 207
 208 def docker_cleanup():
 209     print("Please wait for docker to clean up")
 210     call("docker stop satest", shell=True)
 211
 212
 213 def main():
 214     parser = argparse.ArgumentParser()
 215     subparsers = parser.add_subparsers()
 216
 217     # add subcommand
 218     add_parser = subparsers.add_parser(
 219         "add", help="Add a new project for the analyzer testing."
 220     )
 221     # TODO: Add an option not to build.
 222     # TODO: Set the path to the Repository directory.
 223     add_parser.add_argument("name", nargs=1, help="Name of the new project")
 224     add_parser.add_argument(
 225         "--mode",
 226         action="store",
 227         default=1,
 228         type=int,
 229         choices=[0, 1, 2],
 230         help="Build mode: 0 for single file project, "
 231         "1 for scan_build, "
 232         "2 for single file c++11 project",
 233     )
 234     add_parser.add_argument(
 235         "--source",
 236         action="store",
 237         default="script",
 238         choices=["script", "git", "zip"],
 239         help="Source type of the new project: "
 240         "'git' for getting from git "
 241         "(please provide --origin and --commit), "
 242         "'zip' for unpacking source from a zip file, "
 243         "'script' for downloading source by running "
 244         "a custom script",
 245     )
 246     add_parser.add_argument(
 247         "--origin", action="store", default="", help="Origin link for a git repository"
 248     )
 249     add_parser.add_argument(
 250         "--commit", action="store", default="", help="Git hash for a commit to checkout"
 251     )
 252     add_parser.set_defaults(func=add)
 253
 254     # build subcommand
 255     build_parser = subparsers.add_parser(
 256         "build",
 257         help="Build projects from the project map and compare results with "
 258         "the reference.",
 259     )
 260     build_parser.add_argument(
 261         "--strictness",
 262         dest="strictness",
 263         type=int,
 264         default=0,
 265         help="0 to fail on runtime errors, 1 to fail "
 266         "when the number of found bugs are different "
 267         "from the reference, 2 to fail on any "
 268         "difference from the reference. Default is 0.",
 269     )
 270     build_parser.add_argument(
 271         "-r",
 272         dest="regenerate",
 273         action="store_true",
 274         default=False,
 275         help="Regenerate reference output.",
 276     )
 277     build_parser.add_argument(
 278         "--override-compiler",
 279         action="store_true",
 280         default=False,
 281         help="Call scan-build with " "--override-compiler option.",
 282     )
 283     build_parser.add_argument(
 284         "-j",
 285         "--jobs",
 286         dest="jobs",
 287         type=int,
 288         default=0,
 289         help="Number of projects to test concurrently",
 290     )
 291     build_parser.add_argument(
 292         "--extra-analyzer-config",
 293         dest="extra_analyzer_config",
 294         type=str,
 295         default="",
 296         help="Arguments passed to to -analyzer-config",
 297     )
 298     build_parser.add_argument(
 299         "--extra-checkers",
 300         dest="extra_checkers",
 301         type=str,
 302         default="",
 303         help="Extra checkers to enable",
 304     )
 305     build_parser.add_argument(
 306         "--projects",
 307         action="store",
 308         default="",
 309         help="Comma-separated list of projects to test",
 310     )
 311     build_parser.add_argument(
 312         "--max-size",
 313         action="store",
 314         default=None,
 315         help="Maximum size for the projects to test",
 316     )
 317     build_parser.add_argument("-v", "--verbose", action="count", default=0)
 318     build_parser.set_defaults(func=build)
 319
 320     # compare subcommand
 321     cmp_parser = subparsers.add_parser(
 322         "compare",
 323         help="Comparing two static analyzer runs in terms of "
 324         "reported warnings and execution time statistics.",
 325     )
 326     cmp_parser.add_argument(
 327         "--root-old",
 328         dest="root_old",
 329         help="Prefix to ignore on source files for " "OLD directory",
 330         action="store",
 331         type=str,
 332         default="",
 333     )
 334     cmp_parser.add_argument(
 335         "--root-new",
 336         dest="root_new",
 337         help="Prefix to ignore on source files for " "NEW directory",
 338         action="store",
 339         type=str,
 340         default="",
 341     )
 342     cmp_parser.add_argument(
 343         "--verbose-log",
 344         dest="verbose_log",
 345         help="Write additional information to LOG " "[default=None]",
 346         action="store",
 347         type=str,
 348         default=None,
 349         metavar="LOG",
 350     )
 351     cmp_parser.add_argument(
 352         "--stats-only",
 353         action="store_true",
 354         dest="stats_only",
 355         default=False,
 356         help="Only show statistics on reports",
 357     )
 358     cmp_parser.add_argument(
 359         "--show-stats",
 360         action="store_true",
 361         dest="show_stats",
 362         default=False,
 363         help="Show change in statistics",
 364     )
 365     cmp_parser.add_argument(
 366         "--histogram",
 367         action="store",
 368         default=None,
 369         help="Show histogram of paths differences. " "Requires matplotlib",
 370     )
 371     cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
 372     cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
 373     cmp_parser.set_defaults(func=compare)
 374
 375     # update subcommand
 376     upd_parser = subparsers.add_parser(
 377         "update",
 378         help="Update static analyzer reference results based on the previous "
 379         "run of SATest build. Assumes that SATest build was just run.",
 380     )
 381     upd_parser.add_argument(
 382         "--git", action="store_true", help="Stage updated results using git."
 383     )
 384     upd_parser.set_defaults(func=update)
 385
 386     # docker subcommand
 387     dock_parser = subparsers.add_parser(
 388         "docker", help="Run regression system in the docker."
 389     )
 390
 391     dock_parser.add_argument(
 392         "--build-image",
 393         action="store_true",
 394         help="Build docker image for running tests.",
 395     )
 396     dock_parser.add_argument(
 397         "--shell", action="store_true", help="Start a shell on docker."
 398     )
 399     dock_parser.add_argument(
 400         "--llvm-project-dir",
 401         action="store",
 402         default=DEFAULT_LLVM_DIR,
 403         help="Path to LLVM source code. Defaults "
 404         "to the repo where this script is located. ",
 405     )
 406     dock_parser.add_argument(
 407         "--build-dir",
 408         action="store",
 409         default="",
 410         help="Path to a directory where docker should " "build LLVM code.",
 411     )
 412     dock_parser.add_argument(
 413         "--clang-dir",
 414         action="store",
 415         default="",
 416         help="Path to find/install LLVM installation.",
 417     )
 418     dock_parser.add_argument(
 419         "rest",
 420         nargs=argparse.REMAINDER,
 421         default=[],
 422         help="Additional args that will be forwarded " "to the docker's entrypoint.",
 423     )
 424     dock_parser.set_defaults(func=docker)
 425
 426     # benchmark subcommand
 427     bench_parser = subparsers.add_parser(
 428         "benchmark", help="Run benchmarks by building a set of projects multiple times."
 429     )
 430
 431     bench_parser.add_argument(
 432         "-i",
 433         "--iterations",
 434         action="store",
 435         type=int,
 436         default=20,
 437         help="Number of iterations for building each " "project.",
 438     )
 439     bench_parser.add_argument(
 440         "-o",
 441         "--output",
 442         action="store",
 443         default="benchmark.csv",
 444         help="Output csv file for the benchmark results",
 445     )
 446     bench_parser.add_argument(
 447         "--projects",
 448         action="store",
 449         default="",
 450         help="Comma-separated list of projects to test",
 451     )
 452     bench_parser.add_argument(
 453         "--max-size",
 454         action="store",
 455         default=None,
 456         help="Maximum size for the projects to test",
 457     )
 458     bench_parser.set_defaults(func=benchmark)
 459
 460     bench_subparsers = bench_parser.add_subparsers()
 461     bench_compare_parser = bench_subparsers.add_parser(
 462         "compare", help="Compare benchmark runs."
 463     )
 464     bench_compare_parser.add_argument(
 465         "--old",
 466         action="store",
 467         required=True,
 468         help="Benchmark reference results to " "compare agains.",
 469     )
 470     bench_compare_parser.add_argument(
 471         "--new", action="store", required=True, help="New benchmark results to check."
 472     )
 473     bench_compare_parser.add_argument(
 474         "-o", "--output", action="store", required=True, help="Output file for plots."
 475     )
 476     bench_compare_parser.set_defaults(func=benchmark_compare)
 477
 478     args = parser.parse_args()
 479     args.func(parser, args)
 480
 481
 482 if __name__ == "__main__":
 483     main()