llvm/utils/lit/tests/shtest-format.py

   1 # Check the various features of the ShTest format.
   2
   3 # FIXME: this test depends on order of tests
   4 # RUN: rm -f %{inputs}/shtest-format/.lit_test_times.txt
   5
   6 # RUN: rm -f %t.xml
   7 # RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
   8 # RUN: FileCheck < %t.out %s
   9 # RUN: FileCheck --check-prefix=XUNIT < %t.xml %s
  10
  11 # END.
  12
  13 # CHECK: -- Testing:
  14
  15 # CHECK: FAIL: shtest-format :: external_shell/fail.txt
  16 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail.txt' FAILED ***
  17 # CHECK: Command Output (stdout):
  18 # CHECK-NEXT: --
  19 # CHECK-NEXT: line 1: failed test output on stdout
  20 # CHECK-NEXT: line 2: failed test output on stdout
  21 # CHECK: Command Output (stderr):
  22 # CHECK-NEXT: --
  23 # CHECK-NEXT: cat{{(_64)?(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
  24 # CHECK: --
  25
  26 # CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt
  27 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_bad_encoding.txt' FAILED ***
  28 # CHECK: Command Output (stdout):
  29 # CHECK-NEXT: --
  30 # CHECK-NEXT: a line with bad encoding:
  31 # CHECK: --
  32
  33 # CHECK: FAIL: shtest-format :: external_shell/fail_with_control_chars.txt
  34 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_control_chars.txt' FAILED ***
  35 # CHECK: Command Output (stdout):
  36 # CHECK-NEXT: --
  37 # CHECK-NEXT: a line with {{.*}}control characters{{.*}}.
  38 # CHECK: --
  39
  40 # CHECK: PASS: shtest-format :: external_shell/pass.txt
  41
  42 # CHECK: FAIL: shtest-format :: fail.txt
  43 # CHECK-NEXT: *** TEST 'shtest-format :: fail.txt' FAILED ***
  44 # CHECK-NEXT: Script:
  45 # CHECK-NEXT: --
  46 # CHECK-NEXT: printf "line 1
  47 # CHECK-NEXT: false
  48 # CHECK-NEXT: --
  49 # CHECK-NEXT: Exit Code: 1
  50 #
  51 # CHECK: Command Output (stdout):
  52 # CHECK-NEXT: --
  53 # CHECK-NEXT: $ ":" "RUN: at line 1"
  54 # CHECK-NEXT: $ "printf"
  55 # CHECK-NEXT: # command output:
  56 # CHECK-NEXT: line 1: failed test output on stdout
  57 # CHECK-NEXT: line 2: failed test output on stdout
  58
  59 # CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
  60 # CHECK: PASS: shtest-format :: pass.txt
  61 # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
  62 # CHECK: PASS: shtest-format :: requires-present.txt
  63 # CHECK: UNRESOLVED: shtest-format :: requires-star.txt
  64 # CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
  65 # CHECK: PASS: shtest-format :: unsupported-expr-false.txt
  66 # CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
  67 # CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
  68 # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
  69 # CHECK: PASS: shtest-format :: xfail-expr-false.txt
  70 # CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
  71 # CHECK: XFAIL: shtest-format :: xfail-feature.txt
  72 # CHECK: XFAIL: shtest-format :: xfail-target.txt
  73 # CHECK: XFAIL: shtest-format :: xfail.txt
  74 # CHECK: XPASS: shtest-format :: xpass.txt
  75 # CHECK-NEXT: *** TEST 'shtest-format :: xpass.txt' FAILED ***
  76 # CHECK-NEXT: Script
  77 # CHECK-NEXT: --
  78 # CHECK-NEXT: true
  79 # CHECK-NEXT: --
  80
  81 # CHECK: Failed Tests (4)
  82 # CHECK: shtest-format :: external_shell/fail.txt
  83 # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
  84 # CHECK: shtest-format :: external_shell/fail_with_control_chars.txt
  85 # CHECK: shtest-format :: fail.txt
  86
  87 # CHECK: Unexpectedly Passed Tests (1)
  88 # CHECK: shtest-format :: xpass.txt
  89
  90 # CHECK: Testing Time:
  91 # CHECK: Unsupported        : 4
  92 # CHECK: Passed             : 6
  93 # CHECK: Expectedly Failed  : 4
  94 # CHECK: Unresolved         : 3
  95 # CHECK: Failed             : 4
  96 # CHECK: Unexpectedly Passed: 1
  97
  98
  99 # XUNIT: <?xml version="1.0" encoding="UTF-8"?>
 100 # XUNIT-NEXT: <testsuites time="{{[0-9.]+}}">
 101 # XUNIT-NEXT: <testsuite name="shtest-format" tests="22" failures="8" skipped="4">
 102
 103 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 104 # XUNIT-NEXT: <failure{{[ ]*}}>
 105 # XUNIT: </failure>
 106 # XUNIT-NEXT: </testcase>
 107
 108
 109 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_bad_encoding.txt" time="{{[0-9]+\.[0-9]+}}">
 110 # XUNIT-NEXT: <failure{{[ ]*}}>
 111 # XUNIT: </failure>
 112 # XUNIT-NEXT: </testcase>
 113
 114 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_control_chars.txt" time="{{[0-9]+\.[0-9]+}}">
 115 # XUNIT-NEXT: <failure><![CDATA[Script:
 116 # XUNIT: Command Output (stdout):
 117 # XUNIT-NEXT: --
 118 # XUNIT-NEXT: a line with [2;30;41mcontrol characters[0m.
 119 # XUNIT: </failure>
 120 # XUNIT-NEXT: </testcase>
 121
 122 # XUNIT: <testcase classname="shtest-format.external_shell" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 123
 124 # XUNIT: <testcase classname="shtest-format.shtest-format" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 125 # XUNIT-NEXT: <failure{{[ ]*}}>
 126 # XUNIT: </failure>
 127 # XUNIT-NEXT: </testcase>
 128
 129 # XUNIT: <testcase classname="shtest-format.shtest-format" name="no-test-line.txt" time="{{[0-9]+\.[0-9]+}}">
 130 # XUNIT-NEXT: <failure{{[ ]*}}>
 131 # XUNIT: </failure>
 132 # XUNIT-NEXT: </testcase>
 133
 134 # XUNIT: <testcase classname="shtest-format.shtest-format" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 135
 136 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-missing.txt" time="{{[0-9]+\.[0-9]+}}">
 137 # XUNIT-NEXT:<skipped message="Missing required feature(s): a-missing-feature"/>
 138
 139 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-present.txt" time="{{[0-9]+\.[0-9]+}}"/>
 140
 141 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-star.txt" time="{{[0-9]+\.[0-9]+}}">
 142 # XUNIT-NEXT: <failure{{[ ]*}}>
 143 # XUNIT: </failure>
 144 # XUNIT-NEXT: </testcase>
 145
 146
 147 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-triple.txt" time="{{[0-9]+\.[0-9]+}}">
 148 # XUNIT-NEXT:<skipped message="Missing required feature(s): x86_64"/>
 149
 150 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 151
 152 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-true.txt" time="{{[0-9]+\.[0-9]+}}">
 153 # XUNIT-NEXT:<skipped message="Unsupported configuration"/>
 154
 155 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-star.txt" time="{{[0-9]+\.[0-9]+}}">
 156 # XUNIT-NEXT: <failure{{[ ]*}}>
 157 # XUNIT: </failure>
 158 # XUNIT-NEXT: </testcase>
 159
 160 # XUNIT: <testcase classname="shtest-format.unsupported_dir" name="some-test.txt" time="{{[0-9]+\.[0-9]+}}">
 161 # XUNIT-NEXT:<skipped message="Unsupported configuration"/>
 162
 163 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 164
 165 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-true.txt" time="{{[0-9]+\.[0-9]+}}"/>
 166
 167 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-feature.txt" time="{{[0-9]+\.[0-9]+}}"/>
 168
 169 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-target.txt" time="{{[0-9]+\.[0-9]+}}"/>
 170
 171 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail.txt" time="{{[0-9]+\.[0-9]+}}"/>
 172
 173 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xpass.txt" time="{{[0-9]+\.[0-9]+}}">
 174 # XUNIT-NEXT: <failure{{[ ]*}}>
 175 # XUNIT: </failure>
 176 # XUNIT-NEXT: </testcase>
 177
 178 # XUNIT: </testsuite>
 179 # XUNIT-NEXT: </testsuites>