llvm/utils/lit/tests/shtest-format.py

   1 # Check the various features of the ShTest format.
   2
   3 # RUN: rm -f %t.xml
   4 # RUN: not %{lit} -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
   5 # RUN: FileCheck < %t.out %s
   6 # RUN: FileCheck --check-prefix=XUNIT < %t.xml %s
   7
   8 # END.
   9
  10 # CHECK: -- Testing:
  11
  12 # CHECK: FAIL: shtest-format :: external_shell/fail.txt
  13 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail.txt' FAILED ***
  14 # CHECK: Command Output (stdout):
  15 # CHECK-NEXT: --
  16 # CHECK-NEXT: line 1: failed test output on stdout
  17 # CHECK-NEXT: line 2: failed test output on stdout
  18 # CHECK: Command Output (stderr):
  19 # CHECK-NEXT: --
  20 # CHECK-NOT: --
  21 # CHECK: cat{{(_64)?(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
  22 # CHECK: --
  23
  24 # CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt
  25 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_bad_encoding.txt' FAILED ***
  26 # CHECK: Command Output (stdout):
  27 # CHECK-NEXT: --
  28 # CHECK-NEXT: a line with bad encoding:
  29 # CHECK: --
  30
  31 # CHECK: FAIL: shtest-format :: external_shell/fail_with_control_chars.txt
  32 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_control_chars.txt' FAILED ***
  33 # CHECK: Command Output (stdout):
  34 # CHECK-NEXT: --
  35 # CHECK-NEXT: a line with {{.*}}control characters{{.*}}.
  36 # CHECK: --
  37
  38 # CHECK: PASS: shtest-format :: external_shell/pass.txt
  39
  40 #       CHECK: FAIL: shtest-format :: fail.txt
  41 #  CHECK-NEXT: *** TEST 'shtest-format :: fail.txt' FAILED ***
  42 #  CHECK-NEXT: Exit Code: 1
  43 # CHECK-EMPTY:
  44 #  CHECK-NEXT: Command Output (stdout):
  45 #  CHECK-NEXT: --
  46 #  CHECK-NEXT: # RUN: at line 1
  47 #  CHECK-NEXT: printf "line 1: failed test output on stdout\nline 2: failed test output on stdout"
  48 #  CHECK-NEXT: executed command: printf 'line 1: failed test output on stdout\nline 2: failed test output on stdout'
  49 #  CHECK-NEXT: # .---command stdout------------
  50 #  CHECK-NEXT: # | line 1: failed test output on stdout
  51 #  CHECK-NEXT: # | line 2: failed test output on stdout
  52 #  CHECK-NEXT: # `-----------------------------
  53 #  CHECK-NEXT: # RUN: at line 2
  54 #  CHECK-NEXT: false
  55 #  CHECK-NEXT: # executed command: false
  56 #  CHECK-NEXT: # note: command had no output on stdout or stderr
  57 #  CHECK-NEXT: # error: command failed with exit status: 1
  58 # CHECK-EMPTY:
  59 #  CHECK-NEXT: --
  60
  61
  62 # CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
  63 # CHECK: PASS: shtest-format :: pass.txt
  64 # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
  65 # CHECK: PASS: shtest-format :: requires-present.txt
  66 # CHECK: UNRESOLVED: shtest-format :: requires-star.txt
  67 # CHECK: PASS: shtest-format :: requires-triple.txt
  68 # CHECK: PASS: shtest-format :: unsupported-expr-false.txt
  69 # CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
  70 # CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
  71 # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
  72 # CHECK: PASS: shtest-format :: xfail-expr-false.txt
  73 # CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
  74 # CHECK: XFAIL: shtest-format :: xfail-feature.txt
  75 # CHECK: XFAIL: shtest-format :: xfail-target.txt
  76 # CHECK: XFAIL: shtest-format :: xfail.txt
  77
  78 #       CHECK: XPASS: shtest-format :: xpass.txt
  79 #  CHECK-NEXT: *** TEST 'shtest-format :: xpass.txt' FAILED ***
  80 #  CHECK-NEXT: Exit Code: 0
  81 # CHECK-EMPTY:
  82 #  CHECK-NEXT: Command Output (stdout):
  83 #  CHECK-NEXT: --
  84 #  CHECK-NEXT: # RUN: at line 1
  85 #  CHECK-NEXT: true
  86 #  CHECK-NEXT: # executed command: true
  87 # CHECK-EMPTY:
  88 #  CHECK-NEXT: --
  89
  90 # CHECK: Failed Tests (4)
  91 # CHECK: shtest-format :: external_shell/fail.txt
  92 # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
  93 # CHECK: shtest-format :: external_shell/fail_with_control_chars.txt
  94 # CHECK: shtest-format :: fail.txt
  95
  96 # CHECK: Unexpectedly Passed Tests (1)
  97 # CHECK: shtest-format :: xpass.txt
  98
  99 # CHECK: Testing Time:
 100 # CHECK: Unsupported        : 3
 101 # CHECK: Passed             : 7
 102 # CHECK: Expectedly Failed  : 4
 103 # CHECK: Unresolved         : 3
 104 # CHECK: Failed             : 4
 105 # CHECK: Unexpectedly Passed: 1
 106
 107
 108 # XUNIT: <?xml version="1.0" encoding="UTF-8"?>
 109 # XUNIT-NEXT: <testsuites time="{{[0-9.]+}}">
 110 # XUNIT-NEXT: <testsuite name="shtest-format" tests="22" failures="8" skipped="3">
 111
 112 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 113 # XUNIT-NEXT: <failure{{[ ]*}}>
 114 # XUNIT: </failure>
 115 # XUNIT-NEXT: </testcase>
 116
 117
 118 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_bad_encoding.txt" time="{{[0-9]+\.[0-9]+}}">
 119 # XUNIT-NEXT: <failure{{[ ]*}}>
 120 # XUNIT: </failure>
 121 # XUNIT-NEXT: </testcase>
 122
 123 #       XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_control_chars.txt" time="{{[0-9]+\.[0-9]+}}">
 124 #  XUNIT-NEXT: <failure><![CDATA[Exit Code: 1
 125 # XUNIT-EMPTY:
 126 #  XUNIT-NEXT: Command Output (stdout):
 127 #  XUNIT-NEXT: --
 128 #  XUNIT-NEXT: a line with [2;30;41mcontrol characters[0m.
 129 # XUNIT-EMPTY:
 130 #  XUNIT-NEXT: --
 131 #       XUNIT: ]]></failure>
 132 #  XUNIT-NEXT: </testcase>
 133
 134 # XUNIT: <testcase classname="shtest-format.external_shell" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 135
 136 # XUNIT: <testcase classname="shtest-format.shtest-format" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 137 # XUNIT-NEXT: <failure{{[ ]*}}>
 138 # XUNIT: </failure>
 139 # XUNIT-NEXT: </testcase>
 140
 141 # XUNIT: <testcase classname="shtest-format.shtest-format" name="no-test-line.txt" time="{{[0-9]+\.[0-9]+}}">
 142 # XUNIT-NEXT: <failure{{[ ]*}}>
 143 # XUNIT: </failure>
 144 # XUNIT-NEXT: </testcase>
 145
 146 # XUNIT: <testcase classname="shtest-format.shtest-format" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 147
 148 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-missing.txt" time="{{[0-9]+\.[0-9]+}}">
 149 # XUNIT-NEXT:<skipped message="Missing required feature(s): a-missing-feature"/>
 150
 151 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-present.txt" time="{{[0-9]+\.[0-9]+}}"/>
 152
 153 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-star.txt" time="{{[0-9]+\.[0-9]+}}">
 154 # XUNIT-NEXT: <failure{{[ ]*}}>
 155 # XUNIT: </failure>
 156 # XUNIT-NEXT: </testcase>
 157
 158 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-triple.txt" time="{{[0-9]+\.[0-9]+}}"/>
 159
 160 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 161
 162 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-true.txt" time="{{[0-9]+\.[0-9]+}}">
 163 # XUNIT-NEXT:<skipped message="Unsupported configuration"/>
 164
 165 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-star.txt" time="{{[0-9]+\.[0-9]+}}">
 166 # XUNIT-NEXT: <failure{{[ ]*}}>
 167 # XUNIT: </failure>
 168 # XUNIT-NEXT: </testcase>
 169
 170 # XUNIT: <testcase classname="shtest-format.unsupported_dir" name="some-test.txt" time="{{[0-9]+\.[0-9]+}}">
 171 # XUNIT-NEXT:<skipped message="Unsupported configuration"/>
 172
 173 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 174
 175 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-true.txt" time="{{[0-9]+\.[0-9]+}}"/>
 176
 177 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-feature.txt" time="{{[0-9]+\.[0-9]+}}"/>
 178
 179 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-target.txt" time="{{[0-9]+\.[0-9]+}}"/>
 180
 181 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail.txt" time="{{[0-9]+\.[0-9]+}}"/>
 182
 183 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xpass.txt" time="{{[0-9]+\.[0-9]+}}">
 184 # XUNIT-NEXT: <failure{{[ ]*}}>
 185 # XUNIT: </failure>
 186 # XUNIT-NEXT: </testcase>
 187
 188 # XUNIT: </testsuite>
 189 # XUNIT-NEXT: </testsuites>