llvm/utils/lit/tests/shtest-format.py

   1 # Check the various features of the ShTest format.
   2 #
   3 # RUN: rm -f %t.xml
   4 # RUN: not %{lit} -j 1 -v %{inputs}/shtest-format --xunit-xml-output %t.xml > %t.out
   5 # RUN: FileCheck < %t.out %s
   6 # RUN: FileCheck --check-prefix=XUNIT < %t.xml %s
   7
   8 # END.
   9
  10 # CHECK: -- Testing:
  11
  12 # CHECK: PASS: shtest-format :: argv0.txt
  13 # CHECK: FAIL: shtest-format :: external_shell/fail.txt
  14 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail.txt' FAILED ***
  15 # CHECK: Command Output (stdout):
  16 # CHECK-NEXT: --
  17 # CHECK-NEXT: line 1: failed test output on stdout
  18 # CHECK-NEXT: line 2: failed test output on stdout
  19 # CHECK: Command Output (stderr):
  20 # CHECK-NEXT: --
  21 # CHECK-NEXT: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
  22 # CHECK: --
  23
  24 # CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt
  25 # CHECK-NEXT: *** TEST 'shtest-format :: external_shell/fail_with_bad_encoding.txt' FAILED ***
  26 # CHECK: Command Output (stdout):
  27 # CHECK-NEXT: --
  28 # CHECK-NEXT: a line with bad encoding:
  29 # CHECK: --
  30
  31 # CHECK: PASS: shtest-format :: external_shell/pass.txt
  32
  33 # CHECK: FAIL: shtest-format :: fail.txt
  34 # CHECK-NEXT: *** TEST 'shtest-format :: fail.txt' FAILED ***
  35 # CHECK-NEXT: Script:
  36 # CHECK-NEXT: --
  37 # CHECK-NEXT: printf "line 1
  38 # CHECK-NEXT: false
  39 # CHECK-NEXT: --
  40 # CHECK-NEXT: Exit Code: 1
  41 #
  42 # CHECK: Command Output (stdout):
  43 # CHECK-NEXT: --
  44 # CHECK-NEXT: $ ":" "RUN: at line 1"
  45 # CHECK-NEXT: $ "printf"
  46 # CHECK-NEXT: # command output:
  47 # CHECK-NEXT: line 1: failed test output on stdout
  48 # CHECK-NEXT: line 2: failed test output on stdout
  49
  50 # CHECK: UNRESOLVED: shtest-format :: no-test-line.txt
  51 # CHECK: PASS: shtest-format :: pass.txt
  52 # CHECK: UNSUPPORTED: shtest-format :: requires-missing.txt
  53 # CHECK: PASS: shtest-format :: requires-present.txt
  54 # CHECK: UNRESOLVED: shtest-format :: requires-star.txt
  55 # CHECK: UNSUPPORTED: shtest-format :: requires-triple.txt
  56 # CHECK: PASS: shtest-format :: unsupported-expr-false.txt
  57 # CHECK: UNSUPPORTED: shtest-format :: unsupported-expr-true.txt
  58 # CHECK: UNRESOLVED: shtest-format :: unsupported-star.txt
  59 # CHECK: UNSUPPORTED: shtest-format :: unsupported_dir/some-test.txt
  60 # CHECK: PASS: shtest-format :: xfail-expr-false.txt
  61 # CHECK: XFAIL: shtest-format :: xfail-expr-true.txt
  62 # CHECK: XFAIL: shtest-format :: xfail-feature.txt
  63 # CHECK: XFAIL: shtest-format :: xfail-target.txt
  64 # CHECK: XFAIL: shtest-format :: xfail.txt
  65 # CHECK: XPASS: shtest-format :: xpass.txt
  66 # CHECK-NEXT: *** TEST 'shtest-format :: xpass.txt' FAILED ***
  67 # CHECK-NEXT: Script
  68 # CHECK-NEXT: --
  69 # CHECK-NEXT: true
  70 # CHECK-NEXT: --
  71 # CHECK: Testing Time
  72
  73 # CHECK: Unexpected Passing Tests (1)
  74 # CHECK: shtest-format :: xpass.txt
  75
  76 # CHECK: Failing Tests (3)
  77 # CHECK: shtest-format :: external_shell/fail.txt
  78 # CHECK: shtest-format :: external_shell/fail_with_bad_encoding.txt
  79 # CHECK: shtest-format :: fail.txt
  80
  81 # CHECK: Expected Passes    : 7
  82 # CHECK: Expected Failures  : 4
  83 # CHECK: Unsupported Tests  : 4
  84 # CHECK: Unresolved Tests   : 3
  85 # CHECK: Unexpected Passes  : 1
  86 # CHECK: Unexpected Failures: 3
  87
  88
  89 # XUNIT: <?xml version="1.0" encoding="UTF-8" ?>
  90 # XUNIT-NEXT: <testsuites>
  91 # XUNIT-NEXT: <testsuite name="shtest-format" tests="22" failures="7" skipped="4">
  92
  93 # XUNIT: <testcase classname="shtest-format.shtest-format" name="argv0.txt" time="{{[0-9]+\.[0-9]+}}"/>
  94
  95 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
  96 # XUNIT-NEXT: <failure{{[ ]*}}>
  97 # XUNIT: </failure>
  98 # XUNIT-NEXT: </testcase>
  99
 100
 101 # XUNIT: <testcase classname="shtest-format.external_shell" name="fail_with_bad_encoding.txt" time="{{[0-9]+\.[0-9]+}}">
 102 # XUNIT-NEXT: <failure{{[ ]*}}>
 103 # XUNIT: </failure>
 104 # XUNIT-NEXT: </testcase>
 105
 106 # XUNIT: <testcase classname="shtest-format.external_shell" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 107
 108 # XUNIT: <testcase classname="shtest-format.shtest-format" name="fail.txt" time="{{[0-9]+\.[0-9]+}}">
 109 # XUNIT-NEXT: <failure{{[ ]*}}>
 110 # XUNIT: </failure>
 111 # XUNIT-NEXT: </testcase>
 112
 113 # XUNIT: <testcase classname="shtest-format.shtest-format" name="no-test-line.txt" time="{{[0-9]+\.[0-9]+}}">
 114 # XUNIT-NEXT: <failure{{[ ]*}}>
 115 # XUNIT: </failure>
 116 # XUNIT-NEXT: </testcase>
 117
 118 # XUNIT: <testcase classname="shtest-format.shtest-format" name="pass.txt" time="{{[0-9]+\.[0-9]+}}"/>
 119
 120 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-missing.txt" time="{{[0-9]+\.[0-9]+}}">
 121 # XUNIT-NEXT:<skipped message="Skipping because of: a-missing-feature" />
 122
 123 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-present.txt" time="{{[0-9]+\.[0-9]+}}"/>
 124
 125 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-star.txt" time="{{[0-9]+\.[0-9]+}}">
 126 # XUNIT-NEXT: <failure{{[ ]*}}>
 127 # XUNIT: </failure>
 128 # XUNIT-NEXT: </testcase>
 129
 130
 131 # XUNIT: <testcase classname="shtest-format.shtest-format" name="requires-triple.txt" time="{{[0-9]+\.[0-9]+}}">
 132 # XUNIT-NEXT:<skipped message="Skipping because of: x86_64" />
 133
 134 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 135
 136 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-expr-true.txt" time="{{[0-9]+\.[0-9]+}}">
 137 # XUNIT-NEXT:<skipped message="Skipping because of configuration." />
 138
 139 # XUNIT: <testcase classname="shtest-format.shtest-format" name="unsupported-star.txt" time="{{[0-9]+\.[0-9]+}}">
 140 # XUNIT-NEXT: <failure{{[ ]*}}>
 141 # XUNIT: </failure>
 142 # XUNIT-NEXT: </testcase>
 143
 144 # XUNIT: <testcase classname="shtest-format.unsupported_dir" name="some-test.txt" time="{{[0-9]+\.[0-9]+}}">
 145 # XUNIT-NEXT:<skipped message="Skipping because of configuration." />
 146
 147 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-false.txt" time="{{[0-9]+\.[0-9]+}}"/>
 148
 149 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-expr-true.txt" time="{{[0-9]+\.[0-9]+}}"/>
 150
 151 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-feature.txt" time="{{[0-9]+\.[0-9]+}}"/>
 152
 153 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail-target.txt" time="{{[0-9]+\.[0-9]+}}"/>
 154
 155 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xfail.txt" time="{{[0-9]+\.[0-9]+}}"/>
 156
 157 # XUNIT: <testcase classname="shtest-format.shtest-format" name="xpass.txt" time="{{[0-9]+\.[0-9]+}}">
 158 # XUNIT-NEXT: <failure{{[ ]*}}>
 159 # XUNIT: </failure>
 160 # XUNIT-NEXT: </testcase>
 161
 162 # XUNIT: </testsuite>
 163 # XUNIT-NEXT: </testsuites>