llvm/utils/lit/tests/discovery.py

   1 # Check the basic discovery process, including a sub-suite.
   2 #
   3 # RUN: %{lit} %{inputs}/discovery \
   4 # RUN:   --debug --show-tests --show-suites \
   5 # RUN:   -v > %t.out 2> %t.err
   6 # RUN: FileCheck --check-prefix=CHECK-BASIC-OUT < %t.out %s
   7 # RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s
   8 #
   9 # CHECK-BASIC-ERR: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'
  10 # CHECK-BASIC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'
  11 # CHECK-BASIC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'
  12 #
  13 # CHECK-BASIC-OUT: -- Test Suites --
  14 # CHECK-BASIC-OUT:   sub-suite - 2 tests
  15 # CHECK-BASIC-OUT:     Source Root: {{.*[/\\]discovery[/\\]subsuite$}}
  16 # CHECK-BASIC-OUT:     Exec Root  : {{.*[/\\]discovery[/\\]subsuite$}}
  17 # CHECK-BASIC-OUT:   top-level-suite - 3 tests
  18 # CHECK-BASIC-OUT:     Source Root: {{.*[/\\]discovery$}}
  19 # CHECK-BASIC-OUT:     Exec Root  : {{.*[/\\]discovery$}}
  20 # CHECK-BASIC-OUT:     Available Features: feature1 feature2
  21 # CHECK-BASIC-OUT:     Available Substitutions: %key1 => value1
  22 # CHECK-BASIC-OUT:                              %key2 => value2
  23 #
  24 # CHECK-BASIC-OUT: -- Available Tests --
  25 # CHECK-BASIC-OUT: sub-suite :: test-one
  26 # CHECK-BASIC-OUT: sub-suite :: test-two
  27 # CHECK-BASIC-OUT: top-level-suite :: subdir/test-three
  28 # CHECK-BASIC-OUT: top-level-suite :: test-one
  29 # CHECK-BASIC-OUT: top-level-suite :: test-two
  30
  31 # RUN: %{lit} %{inputs}/discovery \
  32 # RUN:   -v > %t.out 2> %t.err
  33 # RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s
  34 #
  35 # CHECK-PERCENTAGES-OUT:  Total Discovered Tests: {{[0-9]*}}
  36 # CHECK-PERCENTAGES-OUT:  Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}}
  37
  38 # Check discovery when providing the special builtin 'config_map'
  39 # RUN: %{python} %{inputs}/config-map-discovery/driver.py \
  40 # RUN:           %{inputs}/config-map-discovery/main-config/lit.cfg \
  41 # RUN:           %{inputs}/config-map-discovery/lit.alt.cfg \
  42 # RUN:           --workers=1 --debug --show-tests --show-suites > %t.out 2> %t.err
  43 # RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-OUT < %t.out %s
  44 # RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-ERR < %t.err %s
  45
  46 # CHECK-CONFIG-MAP-OUT-NOT: ERROR: lit.cfg invoked
  47 # CHECK-CONFIG-MAP-OUT: -- Test Suites --
  48 # CHECK-CONFIG-MAP-OUT:   config-map - 2 tests
  49 # CHECK-CONFIG-MAP-OUT:     Source Root: {{.*[/\\]config-map-discovery[/\\]tests}}
  50 # CHECK-CONFIG-MAP-OUT:     Exec Root  : {{.*[/\\]tests[/\\]Inputs[/\\]config-map-discovery}}
  51 # CHECK-CONFIG-MAP-OUT: -- Available Tests --
  52 # CHECK-CONFIG-MAP-OUT-NOT: invalid-test.txt
  53 # CHECK-CONFIG-MAP-OUT:   config-map :: test1.txt
  54 # CHECK-CONFIG-MAP-OUT:   config-map :: test2.txt
  55
  56 # CHECK-CONFIG-MAP-ERR: loading suite config '{{.*}}lit.alt.cfg'
  57 # CHECK-CONFIG-MAP-ERR: loaded config '{{.*}}lit.alt.cfg'
  58 # CHECK-CONFIG-MAP-ERR: resolved input '{{.*(/|\\\\)config-map-discovery(/|\\\\)main-config}}' to 'config-map'::()
  59
  60
  61 # Check discovery when tests are named directly.
  62 #
  63 # RUN: %{lit} \
  64 # RUN:     %{inputs}/discovery/subdir/test-three.py \
  65 # RUN:     %{inputs}/discovery/subsuite/test-one.txt \
  66 # RUN:   --show-tests --show-suites -v > %t.out
  67 # RUN: FileCheck --check-prefix=CHECK-DIRECT-TEST < %t.out %s
  68 #
  69 # CHECK-DIRECT-TEST: -- Available Tests --
  70 # CHECK-DIRECT-TEST: sub-suite :: test-one
  71 # CHECK-DIRECT-TEST: top-level-suite :: subdir/test-three
  72
  73 # Check discovery when config files end in .py
  74 # RUN: %{lit} %{inputs}/py-config-discovery \
  75 # RUN:   --debug --show-tests --show-suites \
  76 # RUN:   -v > %t.out 2> %t.err
  77 # RUN: FileCheck --check-prefix=CHECK-PYCONFIG-OUT < %t.out %s
  78 # RUN: FileCheck --check-prefix=CHECK-PYCONFIG-ERR < %t.err %s
  79 #
  80 # CHECK-PYCONFIG-ERR: loading suite config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}'
  81 # CHECK-PYCONFIG-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'
  82 # CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'
  83 # CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}'
  84 # CHECK-PYCONFIG-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'
  85 # CHECK-PYCONFIG-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'
  86 #
  87 # CHECK-PYCONFIG-OUT: -- Test Suites --
  88 # CHECK-PYCONFIG-OUT:   sub-suite - 2 tests
  89 # CHECK-PYCONFIG-OUT:     Source Root: {{.*[/\\]discovery[/\\]subsuite$}}
  90 # CHECK-PYCONFIG-OUT:     Exec Root  : {{.*[/\\]discovery[/\\]subsuite$}}
  91 # CHECK-PYCONFIG-OUT:   top-level-suite - 3 tests
  92 # CHECK-PYCONFIG-OUT:     Source Root: {{.*[/\\]discovery$}}
  93 # CHECK-PYCONFIG-OUT:     Exec Root  : {{.*[/\\]py-config-discovery$}}
  94 #
  95 # CHECK-PYCONFIG-OUT: -- Available Tests --
  96 # CHECK-PYCONFIG-OUT: sub-suite :: test-one
  97 # CHECK-PYCONFIG-OUT: sub-suite :: test-two
  98 # CHECK-PYCONFIG-OUT: top-level-suite :: subdir/test-three
  99 # CHECK-PYCONFIG-OUT: top-level-suite :: test-one
 100 # CHECK-PYCONFIG-OUT: top-level-suite :: test-two
 101
 102 # Check discovery when using an exec path.
 103 #
 104 # RUN: %{lit} %{inputs}/exec-discovery \
 105 # RUN:   --debug --show-tests --show-suites \
 106 # RUN:   -v > %t.out 2> %t.err
 107 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-OUT < %t.out %s
 108 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-ERR < %t.err %s
 109 #
 110 # CHECK-ASEXEC-ERR: loading suite config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}'
 111 # CHECK-ASEXEC-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'
 112 # CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}'
 113 # CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}'
 114 # CHECK-ASEXEC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}'
 115 # CHECK-ASEXEC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}'
 116 #
 117 # CHECK-ASEXEC-OUT: -- Test Suites --
 118 # CHECK-ASEXEC-OUT:   sub-suite - 2 tests
 119 # CHECK-ASEXEC-OUT:     Source Root: {{.*[/\\]discovery[/\\]subsuite$}}
 120 # CHECK-ASEXEC-OUT:     Exec Root  : {{.*[/\\]discovery[/\\]subsuite$}}
 121 # CHECK-ASEXEC-OUT:   top-level-suite - 3 tests
 122 # CHECK-ASEXEC-OUT:     Source Root: {{.*[/\\]discovery$}}
 123 # CHECK-ASEXEC-OUT:     Exec Root  : {{.*[/\\]exec-discovery$}}
 124 #
 125 # CHECK-ASEXEC-OUT: -- Available Tests --
 126 # CHECK-ASEXEC-OUT: sub-suite :: test-one
 127 # CHECK-ASEXEC-OUT: sub-suite :: test-two
 128 # CHECK-ASEXEC-OUT: top-level-suite :: subdir/test-three
 129 # CHECK-ASEXEC-OUT: top-level-suite :: test-one
 130 # CHECK-ASEXEC-OUT: top-level-suite :: test-two
 131
 132 # Check discovery when tests are named directly.
 133 #
 134 # FIXME: Note that using a path into a subsuite doesn't work correctly here.
 135 #
 136 # RUN: %{lit} \
 137 # RUN:     %{inputs}/exec-discovery/subdir/test-three.py \
 138 # RUN:   --show-tests --show-suites -v > %t.out
 139 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-DIRECT-TEST < %t.out %s
 140 #
 141 # CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --
 142 # CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three
 143
 144 # Check that an error is emitted when the directly named test does not satisfy
 145 # the test config's requirements.
 146 #
 147 # RUN: not %{lit} \
 148 # RUN:     %{inputs}/discovery/test.not-txt 2>%t.err
 149 # RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s
 150 #
 151 # CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests
 152 # CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s)
 153
 154 # Check that a standalone test with no suffixes set is run without any errors.
 155 #
 156 # RUN: %{lit} %{inputs}/standalone-tests/true.txt > %t.out
 157 # RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
 158 #
 159 # CHECK-STANDALONE: PASS: Standalone tests :: true.txt
 160
 161 # Check that an error is produced if suffixes variable is set for a suite with
 162 # standalone tests.
 163 #
 164 # RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes 2> %t.err
 165 # RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
 166 #
 167 # CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
 168
 169 # Check that an error is produced if excludes variable is set for a suite with
 170 # standalone tests.
 171 #
 172 # RUN: not %{lit} %{inputs}/standalone-tests-with-excludes 2> %t.err
 173 # RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
 174 #
 175 # CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
 176
 177 # Check that no discovery is done for testsuite with standalone tests.
 178 #
 179 # RUN: not %{lit} %{inputs}/standalone-tests 2>%t.err
 180 # RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
 181 #
 182 # CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
 183
 184 # Check that a single file path can result in multiple tests being discovered if
 185 # the test format implements those semantics.
 186 #
 187 # RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out
 188 # RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s
 189 #
 190 # CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test
 191 # CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test
 192
 193 # Check that we don't recurse infinitely when loading an site specific test
 194 # suite located inside the test source root.
 195 #
 196 # RUN: %{lit} \
 197 # RUN:     %{inputs}/exec-discovery-in-tree/obj/ \
 198 # RUN:   --show-tests --show-suites -v > %t.out
 199 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
 200 #
 201 # Try it again after cd'ing into the test suite using a short relative path.
 202 #
 203 # RUN: cd %{inputs}/exec-discovery-in-tree/obj/
 204 # RUN: %{lit} . \
 205 # RUN:   --show-tests --show-suites -v > %t.out
 206 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
 207 #
 208 #      CHECK-ASEXEC-INTREE:   exec-discovery-in-tree-suite - 1 tests
 209 # CHECK-ASEXEC-INTREE-NEXT:     Source Root: {{.*[/\\]exec-discovery-in-tree$}}
 210 # CHECK-ASEXEC-INTREE-NEXT:     Exec Root  : {{.*[/\\]exec-discovery-in-tree[/\\]obj$}}
 211 # CHECK-ASEXEC-INTREE-NEXT:     Available Features:
 212 # CHECK-ASEXEC-INTREE-NEXT:     Available Substitutions:
 213 # CHECK-ASEXEC-INTREE-NEXT: -- Available Tests --
 214 # CHECK-ASEXEC-INTREE-NEXT: exec-discovery-in-tree-suite :: test-one