utils/lit/tests/selecting.py

   1 # RUN: %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-BASIC %s
   2 # CHECK-BASIC: Testing: 5 tests
   3
   4 # Check that regex-filtering works, is case-insensitive, and can be configured via env var.
   5 #
   6 # RUN: %{lit} --filter 'o[a-z]e' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
   7 # RUN: %{lit} --filter 'O[A-Z]E' %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
   8 # RUN: env LIT_FILTER='o[a-z]e' %{lit} %{inputs}/discovery | FileCheck --check-prefix=CHECK-FILTER %s
   9 # CHECK-FILTER: Testing: 2 of 5 tests
  10
  11 # Check that maximum counts work
  12 #
  13 # RUN: %{lit} --max-tests 3 %{inputs}/discovery | FileCheck --check-prefix=CHECK-MAX %s
  14 # CHECK-MAX: Testing: 3 of 5 tests
  15
  16
  17 # Check that sharding partitions the testsuite in a way that distributes the
  18 # rounding error nicely (i.e. 5/3 => 2 2 1, not 1 1 3 or whatever)
  19 #
  20 # RUN: %{lit} --num-shards 3 --run-shard 1 %{inputs}/discovery >%t.out 2>%t.err
  21 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ERR < %t.err %s
  22 # RUN: FileCheck --check-prefix=CHECK-SHARD0-OUT < %t.out %s
  23 # CHECK-SHARD0-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
  24 # CHECK-SHARD0-OUT: Testing: 2 of 5 tests
  25 #
  26 # RUN: %{lit} --num-shards 3 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
  27 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ERR < %t.err %s
  28 # RUN: FileCheck --check-prefix=CHECK-SHARD1-OUT < %t.out %s
  29 # CHECK-SHARD1-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
  30 # CHECK-SHARD1-OUT: Testing: 2 of 5 tests
  31 #
  32 # RUN: %{lit} --num-shards 3 --run-shard 3 %{inputs}/discovery >%t.out 2>%t.err
  33 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ERR < %t.err %s
  34 # RUN: FileCheck --check-prefix=CHECK-SHARD2-OUT < %t.out %s
  35 # CHECK-SHARD2-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
  36 # CHECK-SHARD2-OUT: Testing: 1 of 5 tests
  37
  38
  39 # Check that sharding via env vars works.
  40 #
  41 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=1 %{lit} %{inputs}/discovery >%t.out 2>%t.err
  42 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-ERR < %t.err %s
  43 # RUN: FileCheck --check-prefix=CHECK-SHARD0-ENV-OUT < %t.out %s
  44 # CHECK-SHARD0-ENV-ERR: note: Selecting shard 1/3 = size 2/5 = tests #(3*k)+1 = [1, 4]
  45 # CHECK-SHARD0-ENV-OUT: Testing: 2 of 5 tests
  46 #
  47 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=2 %{lit} %{inputs}/discovery >%t.out 2>%t.err
  48 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-ERR < %t.err %s
  49 # RUN: FileCheck --check-prefix=CHECK-SHARD1-ENV-OUT < %t.out %s
  50 # CHECK-SHARD1-ENV-ERR: note: Selecting shard 2/3 = size 2/5 = tests #(3*k)+2 = [2, 5]
  51 # CHECK-SHARD1-ENV-OUT: Testing: 2 of 5 tests
  52 #
  53 # RUN: env LIT_NUM_SHARDS=3 LIT_RUN_SHARD=3 %{lit} %{inputs}/discovery >%t.out 2>%t.err
  54 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-ERR < %t.err %s
  55 # RUN: FileCheck --check-prefix=CHECK-SHARD2-ENV-OUT < %t.out %s
  56 # CHECK-SHARD2-ENV-ERR: note: Selecting shard 3/3 = size 1/5 = tests #(3*k)+3 = [3]
  57 # CHECK-SHARD2-ENV-OUT: Testing: 1 of 5 tests
  58
  59
  60 # Check that providing more shards than tests results in 1 test per shard
  61 # until we run out, then 0.
  62 #
  63 # RUN: %{lit} --num-shards 100 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
  64 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR1 < %t.err %s
  65 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT1 < %t.out %s
  66 # CHECK-SHARD-BIG-ERR1: note: Selecting shard 2/100 = size 1/5 = tests #(100*k)+2 = [2]
  67 # CHECK-SHARD-BIG-OUT1: Testing: 1 of 5 tests
  68 #
  69 # RUN: %{lit} --num-shards 100 --run-shard 6 %{inputs}/discovery >%t.out 2>%t.err
  70 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR2 < %t.err %s
  71 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT2 < %t.out %s
  72 # CHECK-SHARD-BIG-ERR2: note: Selecting shard 6/100 = size 0/5 = tests #(100*k)+6 = []
  73 # CHECK-SHARD-BIG-OUT2: Testing: 0 of 5 tests
  74 #
  75 # RUN: %{lit} --num-shards 100 --run-shard 50 %{inputs}/discovery >%t.out 2>%t.err
  76 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-ERR3 < %t.err %s
  77 # RUN: FileCheck --check-prefix=CHECK-SHARD-BIG-OUT3 < %t.out %s
  78 # CHECK-SHARD-BIG-ERR3: note: Selecting shard 50/100 = size 0/5 = tests #(100*k)+50 = []
  79 # CHECK-SHARD-BIG-OUT3: Testing: 0 of 5 tests
  80
  81
  82 # Check that range constraints are enforced
  83 #
  84 # RUN: not %{lit} --num-shards 0 --run-shard 2 %{inputs}/discovery >%t.out 2>%t.err
  85 # RUN: FileCheck --check-prefix=CHECK-SHARD-ERR < %t.err %s
  86 # CHECK-SHARD-ERR: error: argument --num-shards: requires positive integer, but found '0'
  87 #
  88 # RUN: not %{lit} --num-shards 3 --run-shard 4 %{inputs}/discovery >%t.out 2>%t.err
  89 # RUN: FileCheck --check-prefix=CHECK-SHARD-ERR2 < %t.err %s
  90 # CHECK-SHARD-ERR2: error: --run-shard must be between 1 and --num-shards (inclusive)