utils/lit/tests/shtest-format.py

   1 # Check the various features of the ShTest format.
   2 #
   3 # RUN: rm -f %t.xml
   4 # RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
   5 # RUN: FileCheck < %t.out %s
   6 # RUN: FileCheck --check-prefix=XUNIT < %t.xml %s
   7
   8 # END.
   9
  10 # CHECK: -- Testing:
  11
  12 # CHECK: PASS: shtest-format :: argv0.txt
  13 # CHECK: FAIL: shtest-format :: external_shell/fail.txt
  14 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail.txt' FAILED ***
  15 # CHECK: Command Output (stdout):
  16 # CHECK-NEXT: --
  17 # CHECK-NEXT: line 1: failed test output on stdout
  18 # CHECK-NEXT: line 2: failed test output on stdout
  19 # CHECK: Command Output (stderr):
  20 # CHECK-NEXT: --
  21 # CHECK-NEXT: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
  22 # CHECK: --
  23
  24 # CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt
  25 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_bad_encoding.txt' FAILED ***
  26 # CHECK: Command Output (stdout):
  27 # CHECK-NEXT: --
  28 # CHECK-NEXT: a line with bad encoding:
  29 # CHECK: --
  30
  31 # CHECK: PASS: shtest-format :: external_shell/pass.txt
  32
  33 # CHECK: FAIL: shtest-format :: fail.txt
  34 # CHECK-NEXT: *** TEST 'shtest-format :: fail.txt' FAILED ***
  35 # CHECK-NEXT: Script:
  36 # CHECK-NEXT: --
  37 # CHECK-NEXT: printf "line 1
  38 # CHECK-NEXT: false
  39 # CHECK-NEXT: --
  40 # CHECK-NEXT: Exit Code: 1
  41 #
  42 # CHECK: Command Output (stdout):
  43 # CHECK-NEXT: --
  44 # CHECK-NEXT: $ ":" "RUN: at line 1"
  45 # CHECK-NEXT: $ "printf"
  46 # CHECK-NEXT: # command output:
  47 # CHECK-NEXT: line 1: failed test output on stdout
  48 # CHECK-NEXT: line 2: failed test output on stdout
  49
  50 # CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
  51 # CHECK: PASS: shtest-format :: pass.txt
  52 # CHECK: UNSUPPORTED: shtest-format :: requires-any-missing.txt
  53 # CHECK: PASS: shtest-format :: requires-any-present.txt
  54 # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
  55 # CHECK: PASS: shtest-format :: requires-present.txt
  56 # CHECK: UNRESOLVED: shtest-format :: requires-star.txt
  57 # CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
  58 # CHECK: PASS: shtest-format :: unsupported-expr-false.txt
  59 # CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
  60 # CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
  61 # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
  62 # CHECK: PASS: shtest-format :: xfail-expr-false.txt
  63 # CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
  64 # CHECK: XFAIL: shtest-format :: xfail-feature.txt
  65 # CHECK: XFAIL: shtest-format :: xfail-target.txt
  66 # CHECK: XFAIL: shtest-format :: xfail.txt
  67 # CHECK: XPASS: shtest-format :: xpass.txt
  68 # CHECK-NEXT: *** TEST 'shtest-format :: xpass.txt' FAILED ***
  69 # CHECK-NEXT: Script
  70 # CHECK-NEXT: --
  71 # CHECK-NEXT: true
  72 # CHECK-NEXT: --
  73 # CHECK: Testing Time
  74
  75 # CHECK: Unexpected Passing Tests (1)
  76 # CHECK: shtest-format :: xpass.txt
  77
  78 # CHECK: Failing Tests (3)
  79 # CHECK: shtest-format :: external_shell/fail.txt
  80 # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
  81 # CHECK: shtest-format :: fail.txt
  82
  83 # CHECK: Expected Passes    : 8
  84 # CHECK: Expected Failures  : 4
  85 # CHECK: Unsupported Tests  : 5
  86 # CHECK: Unresolved Tests   : 3
  87 # CHECK: Unexpected Passes  : 1
  88 # CHECK: Unexpected Failures: 3
  89
  90
  91 # XUNIT: <?xml version="1.0" encoding="UTF-8" ?>
  92 # XUNIT-NEXT: <testsuites>
  93 # XUNIT-NEXT: <testsuite name="shtest-format" tests="24" failures="7" skipped="5">
  94
  95 # XUNIT: <testcase classname="shtest-format.shtest-format" name="argv0.txt" time="{{[0-9]+\.[0-9]+}}"/>
  96
  97 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
  98 # XUNIT-NEXT: <failure{{[ ]*}}>
  99 # XUNIT: </failure>
 100 # XUNIT-NEXT: </testcase>
 101
 102
 103 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_bad_encoding.txt" time="{{[0-9]+\.[0-9]+}}">
 104 # XUNIT-NEXT: <failure{{[ ]*}}>
 105 # XUNIT: </failure>
 106 # XUNIT-NEXT: </testcase>
 107
 108 # XUNIT: <testcase classname="shtest-format.external_shell" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 109
 110 # XUNIT: <testcase classname="shtest-format.shtest-format" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 111 # XUNIT-NEXT: <failure{{[ ]*}}>
 112 # XUNIT: </failure>
 113 # XUNIT-NEXT: </testcase>
 114
 115 # XUNIT: <testcase classname="shtest-format.shtest-format" name="no-test-line.txt" time="{{[0-9]+\.[0-9]+}}">
 116 # XUNIT-NEXT: <failure{{[ ]*}}>
 117 # XUNIT: </failure>
 118 # XUNIT-NEXT: </testcase>
 119
 120 # XUNIT: <testcase classname="shtest-format.shtest-format" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 121
 122 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-any-missing.txt" time="{{[0-9]+\.[0-9]+}}">
 123 # XUNIT-NEXT:<skipped message="Skipping because of: a-missing-feature || a-missing-feature-2" />
 124
 125 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-any-present.txt" time="{{[0-9]+\.[0-9]+}}"/>
 126
 127 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-missing.txt" time="{{[0-9]+\.[0-9]+}}">
 128 # XUNIT-NEXT:<skipped message="Skipping because of: a-missing-feature" />
 129
 130 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-present.txt" time="{{[0-9]+\.[0-9]+}}"/>
 131
 132 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-star.txt" time="{{[0-9]+\.[0-9]+}}">
 133 # XUNIT-NEXT: <failure{{[ ]*}}>
 134 # XUNIT: </failure>
 135 # XUNIT-NEXT: </testcase>
 136
 137
 138 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-triple.txt" time="{{[0-9]+\.[0-9]+}}">
 139 # XUNIT-NEXT:<skipped message="Skipping because of: x86_64" />
 140
 141 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 142
 143 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-true.txt" time="{{[0-9]+\.[0-9]+}}">
 144 # XUNIT-NEXT:<skipped message="Skipping because of configuration." />
 145
 146 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-star.txt" time="{{[0-9]+\.[0-9]+}}">
 147 # XUNIT-NEXT: <failure{{[ ]*}}>
 148 # XUNIT: </failure>
 149 # XUNIT-NEXT: </testcase>
 150
 151 # XUNIT: <testcase classname="shtest-format.unsupported_dir" name="some-test.txt" time="{{[0-9]+\.[0-9]+}}">
 152 # XUNIT-NEXT:<skipped message="Skipping because of configuration." />
 153
 154 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 155
 156 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-true.txt" time="{{[0-9]+\.[0-9]+}}"/>
 157
 158 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-feature.txt" time="{{[0-9]+\.[0-9]+}}"/>
 159
 160 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-target.txt" time="{{[0-9]+\.[0-9]+}}"/>
 161
 162 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail.txt" time="{{[0-9]+\.[0-9]+}}"/>
 163
 164 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xpass.txt" time="{{[0-9]+\.[0-9]+}}">
 165 # XUNIT-NEXT: <failure{{[ ]*}}>
 166 # XUNIT: </failure>
 167 # XUNIT-NEXT: </testcase>
 168
 169 # XUNIT: </testsuite>
 170 # XUNIT-NEXT: </testsuites>