llvm/utils/lit/lit/cl_arguments.py

   1 import argparse
   2 import enum
   3 import os
   4 import shlex
   5 import sys
   6
   7 import lit.reports
   8 import lit.util
   9
  10
  11 @enum.unique
  12 class TestOrder(enum.Enum):
  13     LEXICAL = "lexical"
  14     RANDOM = "random"
  15     SMART = "smart"
  16
  17
  18 def parse_args():
  19     parser = argparse.ArgumentParser(prog="lit", fromfile_prefix_chars="@")
  20     parser.add_argument(
  21         "test_paths",
  22         nargs="+",
  23         metavar="TEST_PATH",
  24         help="File or path to include in the test suite",
  25     )
  26
  27     parser.add_argument(
  28         "--version", action="version", version="%(prog)s " + lit.__version__
  29     )
  30
  31     parser.add_argument(
  32         "-j",
  33         "--threads",
  34         "--workers",
  35         dest="workers",
  36         metavar="N",
  37         help="Number of workers used for testing",
  38         type=_positive_int,
  39         default=lit.util.usable_core_count(),
  40     )
  41     parser.add_argument(
  42         "--config-prefix",
  43         dest="configPrefix",
  44         metavar="NAME",
  45         help="Prefix for 'lit' config files",
  46     )
  47     parser.add_argument(
  48         "-D",
  49         "--param",
  50         dest="user_params",
  51         metavar="NAME=VAL",
  52         help="Add 'NAME' = 'VAL' to the user defined parameters",
  53         action="append",
  54         default=[],
  55     )
  56
  57     format_group = parser.add_argument_group("Output Format")
  58     # FIXME: I find these names very confusing, although I like the
  59     # functionality.
  60     format_group.add_argument(
  61         "-q", "--quiet", help="Suppress no error output", action="store_true"
  62     )
  63     format_group.add_argument(
  64         "-s",
  65         "--succinct",
  66         help="Reduce amount of output."
  67         " Additionally, show a progress bar,"
  68         " unless --no-progress-bar is specified.",
  69         action="store_true",
  70     )
  71     format_group.add_argument(
  72         "-v",
  73         "--verbose",
  74         dest="showOutput",
  75         help="Show test output for failures",
  76         action="store_true",
  77     )
  78     format_group.add_argument(
  79         "-vv",
  80         "--echo-all-commands",
  81         dest="echoAllCommands",
  82         action="store_true",
  83         help="Echo all commands as they are executed to stdout. In case of "
  84         "failure, last command shown will be the failing one.",
  85     )
  86     format_group.add_argument(
  87         "-a",
  88         "--show-all",
  89         dest="showAllOutput",
  90         help="Display all commandlines and output",
  91         action="store_true",
  92     )
  93     format_group.add_argument(
  94         "-o",
  95         "--output",
  96         type=lit.reports.JsonReport,
  97         help="Write test results to the provided path",
  98         metavar="PATH",
  99     )
 100     format_group.add_argument(
 101         "--no-progress-bar",
 102         dest="useProgressBar",
 103         help="Do not use curses based progress bar",
 104         action="store_false",
 105     )
 106
 107     # Note: this does not generate flags for user-defined result codes.
 108     success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]
 109     for code in success_codes:
 110         format_group.add_argument(
 111             "--show-{}".format(code.name.lower()),
 112             dest="shown_codes",
 113             help="Show {} tests ({})".format(code.label.lower(), code.name),
 114             action="append_const",
 115             const=code,
 116             default=[],
 117         )
 118
 119     execution_group = parser.add_argument_group("Test Execution")
 120     execution_group.add_argument(
 121         "--path",
 122         help="Additional paths to add to testing environment",
 123         action="append",
 124         default=[],
 125         type=os.path.abspath,
 126     )
 127     execution_group.add_argument(
 128         "--vg", dest="useValgrind", help="Run tests under valgrind", action="store_true"
 129     )
 130     execution_group.add_argument(
 131         "--vg-leak",
 132         dest="valgrindLeakCheck",
 133         help="Check for memory leaks under valgrind",
 134         action="store_true",
 135     )
 136     execution_group.add_argument(
 137         "--vg-arg",
 138         dest="valgrindArgs",
 139         metavar="ARG",
 140         help="Specify an extra argument for valgrind",
 141         action="append",
 142         default=[],
 143     )
 144     execution_group.add_argument(
 145         "--time-tests",
 146         help="Track elapsed wall time for each test",
 147         action="store_true",
 148     )
 149     execution_group.add_argument(
 150         "--no-execute",
 151         dest="noExecute",
 152         help="Don't execute any tests (assume PASS)",
 153         action="store_true",
 154     )
 155     execution_group.add_argument(
 156         "--xunit-xml-output",
 157         type=lit.reports.XunitReport,
 158         help="Write XUnit-compatible XML test reports to the specified file",
 159     )
 160     execution_group.add_argument(
 161         "--resultdb-output",
 162         type=lit.reports.ResultDBReport,
 163         help="Write LuCI ResuldDB compatible JSON to the specified file",
 164     )
 165     execution_group.add_argument(
 166         "--time-trace-output",
 167         type=lit.reports.TimeTraceReport,
 168         help="Write Chrome tracing compatible JSON to the specified file",
 169     )
 170     execution_group.add_argument(
 171         "--timeout",
 172         dest="maxIndividualTestTime",
 173         help="Maximum time to spend running a single test (in seconds). "
 174         "0 means no time limit. [Default: 0]",
 175         type=_non_negative_int,
 176     )
 177     execution_group.add_argument(
 178         "--max-failures",
 179         help="Stop execution after the given number of failures.",
 180         type=_positive_int,
 181     )
 182     execution_group.add_argument(
 183         "--allow-empty-runs",
 184         help="Do not fail the run if all tests are filtered out",
 185         action="store_true",
 186     )
 187     execution_group.add_argument(
 188         "--ignore-fail",
 189         dest="ignoreFail",
 190         action="store_true",
 191         help="Exit with status zero even if some tests fail",
 192     )
 193     execution_group.add_argument(
 194         "--no-indirectly-run-check",
 195         dest="indirectlyRunCheck",
 196         help="Do not error if a test would not be run if the user had "
 197         "specified the containing directory instead of naming the "
 198         "test directly.",
 199         action="store_false",
 200     )
 201
 202     selection_group = parser.add_argument_group("Test Selection")
 203     selection_group.add_argument(
 204         "--max-tests",
 205         metavar="N",
 206         help="Maximum number of tests to run",
 207         type=_positive_int,
 208     )
 209     selection_group.add_argument(
 210         "--max-time",
 211         dest="timeout",
 212         metavar="N",
 213         help="Maximum time to spend testing (in seconds)",
 214         type=_positive_int,
 215     )
 216     selection_group.add_argument(
 217         "--order",
 218         choices=[x.value for x in TestOrder],
 219         default=TestOrder.SMART,
 220         help="Test order to use (default: smart)",
 221     )
 222     selection_group.add_argument(
 223         "--shuffle",
 224         dest="order",
 225         help="Run tests in random order (DEPRECATED: use --order=random)",
 226         action="store_const",
 227         const=TestOrder.RANDOM,
 228     )
 229     selection_group.add_argument(
 230         "-i",
 231         "--incremental",
 232         help="Run failed tests first (DEPRECATED: use --order=smart)",
 233         action="store_true",
 234     )
 235     selection_group.add_argument(
 236         "--filter",
 237         metavar="REGEX",
 238         type=_case_insensitive_regex,
 239         help="Only run tests with paths matching the given regular expression",
 240         default=os.environ.get("LIT_FILTER", ".*"),
 241     )
 242     selection_group.add_argument(
 243         "--filter-out",
 244         metavar="REGEX",
 245         type=_case_insensitive_regex,
 246         help="Filter out tests with paths matching the given regular expression",
 247         default=os.environ.get("LIT_FILTER_OUT", "^$"),
 248     )
 249     selection_group.add_argument(
 250         "--xfail",
 251         metavar="LIST",
 252         type=_semicolon_list,
 253         help="XFAIL tests with paths in the semicolon separated list",
 254         default=os.environ.get("LIT_XFAIL", ""),
 255     )
 256     selection_group.add_argument(
 257         "--xfail-not",
 258         metavar="LIST",
 259         type=_semicolon_list,
 260         help="do not XFAIL tests with paths in the semicolon separated list",
 261         default=os.environ.get("LIT_XFAIL_NOT", ""),
 262     )
 263     selection_group.add_argument(
 264         "--num-shards",
 265         dest="numShards",
 266         metavar="M",
 267         help="Split testsuite into M pieces and only run one",
 268         type=_positive_int,
 269         default=os.environ.get("LIT_NUM_SHARDS"),
 270     )
 271     selection_group.add_argument(
 272         "--run-shard",
 273         dest="runShard",
 274         metavar="N",
 275         help="Run shard #N of the testsuite",
 276         type=_positive_int,
 277         default=os.environ.get("LIT_RUN_SHARD"),
 278     )
 279
 280     debug_group = parser.add_argument_group("Debug and Experimental Options")
 281     debug_group.add_argument(
 282         "--debug", help="Enable debugging (for 'lit' development)", action="store_true"
 283     )
 284     debug_group.add_argument(
 285         "--show-suites",
 286         help="Show discovered test suites and exit",
 287         action="store_true",
 288     )
 289     debug_group.add_argument(
 290         "--show-tests", help="Show all discovered tests and exit", action="store_true"
 291     )
 292     debug_group.add_argument(
 293         "--show-used-features",
 294         help="Show all features used in the test suite (in XFAIL, UNSUPPORTED and REQUIRES) and exit",
 295         action="store_true",
 296     )
 297
 298     # LIT is special: environment variables override command line arguments.
 299     env_args = shlex.split(os.environ.get("LIT_OPTS", ""))
 300     args = sys.argv[1:] + env_args
 301     opts = parser.parse_args(args)
 302
 303     # Validate command line options
 304     if opts.echoAllCommands:
 305         opts.showOutput = True
 306
 307     if opts.incremental:
 308         print(
 309             "WARNING: --incremental is deprecated. Failing tests now always run first."
 310         )
 311
 312     if opts.numShards or opts.runShard:
 313         if not opts.numShards or not opts.runShard:
 314             parser.error("--num-shards and --run-shard must be used together")
 315         if opts.runShard > opts.numShards:
 316             parser.error("--run-shard must be between 1 and --num-shards (inclusive)")
 317         opts.shard = (opts.runShard, opts.numShards)
 318     else:
 319         opts.shard = None
 320
 321     opts.reports = filter(
 322         None,
 323         [
 324             opts.output,
 325             opts.xunit_xml_output,
 326             opts.resultdb_output,
 327             opts.time_trace_output,
 328         ],
 329     )
 330
 331     return opts
 332
 333
 334 def _positive_int(arg):
 335     return _int(arg, "positive", lambda i: i > 0)
 336
 337
 338 def _non_negative_int(arg):
 339     return _int(arg, "non-negative", lambda i: i >= 0)
 340
 341
 342 def _int(arg, kind, pred):
 343     desc = "requires {} integer, but found '{}'"
 344     try:
 345         i = int(arg)
 346     except ValueError:
 347         raise _error(desc, kind, arg)
 348     if not pred(i):
 349         raise _error(desc, kind, arg)
 350     return i
 351
 352
 353 def _case_insensitive_regex(arg):
 354     import re
 355
 356     try:
 357         return re.compile(arg, re.IGNORECASE)
 358     except re.error as reason:
 359         raise _error("invalid regular expression: '{}', {}", arg, reason)
 360
 361
 362 def _semicolon_list(arg):
 363     return arg.split(";")
 364
 365
 366 def _error(desc, *args):
 367     msg = desc.format(*args)
 368     return argparse.ArgumentTypeError(msg)