external/bsd/kyua-cli/dist/integration/cmd_report_html_test.sh

   1 # Copyright 2012 Google Inc.
   2 # All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 # * Redistributions of source code must retain the above copyright
   9 #   notice, this list of conditions and the following disclaimer.
  10 # * Redistributions in binary form must reproduce the above copyright
  11 #   notice, this list of conditions and the following disclaimer in the
  12 #   documentation and/or other materials provided with the distribution.
  13 # * Neither the name of Google Inc. nor the names of its contributors
  14 #   may be used to endorse or promote products derived from this software
  15 #   without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29
  30 # Executes a mock test suite to generate data in the database.
  31 #
  32 # \param mock_env The value to store in a MOCK variable in the environment.
  33 #     Use this to be able to differentiate executions by inspecting the
  34 #     context of the output.
  35 #
  36 # \return The action identifier of the committed action.
  37 run_tests() {
  38     local mock_env="${1}"
  39
  40     mkdir testsuite
  41     cd testsuite
  42
  43     cat >Kyuafile <<EOF
  44 syntax(2)
  45 test_suite("integration")
  46 atf_test_program{name="simple_all_pass"}
  47 atf_test_program{name="simple_some_fail"}
  48 atf_test_program{name="metadata"}
  49 EOF
  50
  51     utils_cp_helper simple_all_pass .
  52     utils_cp_helper simple_some_fail .
  53     utils_cp_helper metadata .
  54     test -d ../.kyua || mkdir ../.kyua
  55     kyua=$(which kyua)
  56     atf_check -s exit:1 -o save:stdout -e empty env \
  57         HOME="$(pwd)/home" MOCK="${mock_env}" \
  58         "${kyua}" test --store=../.kyua/store.db
  59
  60     action_id=$(grep '^Committed action ' stdout | cut -d ' ' -f 3)
  61     echo "New action is ${action_id}"
  62
  63     cd -
  64     # Ensure the results of 'report' come from the database.
  65     rm -rf testsuite
  66
  67     return "${action_id}"
  68 }
  69
  70
  71 # Ensure a file has a set of strings.
  72 #
  73 # \param file The name of the file to check.
  74 # \param ... List of strings to check.
  75 check_in_file() {
  76     local file="${1}"; shift
  77
  78     while [ ${#} -gt 0 ]; do
  79         echo "Checking for presence of '${1}' in ${file}"
  80         if grep "${1}" "${file}" >/dev/null; then
  81             :
  82         else
  83             atf_fail "Test case output not found in HTML page ${file}"
  84         fi
  85         shift
  86     done
  87 }
  88
  89
  90 # Ensure a file does not have a set of strings.
  91 #
  92 # \param file The name of the file to check.
  93 # \param ... List of strings to check.
  94 check_not_in_file() {
  95     local file="${1}"; shift
  96
  97     while [ ${#} -gt 0 ]; do
  98         echo "Checking for lack of '${1}' in ${file}"
  99         if grep "${1}" "${file}" >/dev/null; then
 100             atf_fail "Spurious test case output found in HTML page"
 101         fi
 102         shift
 103     done
 104 }
 105
 106
 107 utils_test_case default_behavior__ok
 108 default_behavior__ok_body() {
 109     utils_install_timestamp_wrapper
 110
 111     run_tests "mock1"
 112
 113     atf_check -s exit:0 -o ignore -e empty kyua report-html
 114     for f in \
 115         html/index.html \
 116         html/context.html \
 117         html/simple_all_pass_skip.html \
 118         html/simple_some_fail_fail.html
 119     do
 120         test -f "${f}" || atf_fail "Missing ${f}"
 121     done
 122
 123     atf_check -o match:"2 TESTS FAILING" cat html/index.html
 124
 125     check_in_file html/simple_all_pass_skip.html \
 126         "This is the stdout of skip" "This is the stderr of skip"
 127     check_not_in_file html/simple_all_pass_skip.html \
 128         "This is the stdout of pass" "This is the stderr of pass" \
 129         "This is the stdout of fail" "This is the stderr of fail" \
 130         "Test case did not write anything to"
 131
 132     check_in_file html/simple_some_fail_fail.html \
 133         "This is the stdout of fail" "This is the stderr of fail"
 134     check_not_in_file html/simple_some_fail_fail.html \
 135         "This is the stdout of pass" "This is the stderr of pass" \
 136         "This is the stdout of skip" "This is the stderr of skip" \
 137         "Test case did not write anything to"
 138
 139     check_in_file html/metadata_one_property.html \
 140         "description = Does nothing but has one metadata property"
 141     check_not_in_file html/metadata_one_property.html \
 142         "allowed_architectures = some-architecture"
 143
 144     check_in_file html/metadata_many_properties.html \
 145         "allowed_architectures = some-architecture"
 146     check_not_in_file html/metadata_many_properties.html \
 147         "description = Does nothing but has one metadata property"
 148 }
 149
 150
 151 utils_test_case default_behavior__no_actions
 152 default_behavior__no_actions_body() {
 153     kyua db-exec "SELECT * FROM actions"
 154
 155     echo 'kyua: E: No actions in the database.' >experr
 156     atf_check -s exit:2 -o empty -e file:experr kyua report-html
 157 }
 158
 159
 160 utils_test_case default_behavior__no_store
 161 default_behavior__no_store_body() {
 162     atf_check -s exit:2 -o empty \
 163         -e match:"kyua: E: Cannot open '.*/.kyua/store.db': " kyua report-html
 164 }
 165
 166
 167 utils_test_case action__explicit
 168 action__explicit_body() {
 169     run_tests "mock1"; action1=$?
 170     run_tests "mock2"; action2=$?
 171
 172     atf_check -s exit:0 -o ignore -e empty kyua report-html \
 173         --action="${action1}"
 174     grep "action 1" html/index.html || atf_fail "Invalid action in report"
 175     grep "MOCK.*mock1" html/context.html || atf_fail "Invalid context in report"
 176
 177     rm -rf html
 178     atf_check -s exit:0 -o ignore -e empty kyua report-html \
 179         --action="${action2}"
 180     grep "action 2" html/index.html || atf_fail "Invalid action in report"
 181     grep "MOCK.*mock2" html/context.html || atf_fail "Invalid context in report"
 182 }
 183
 184
 185 utils_test_case action__not_found
 186 action__not_found_body() {
 187     kyua db-exec "SELECT * FROM actions"
 188
 189     echo 'kyua: E: Error loading action 514: does not exist.' >experr
 190     atf_check -s exit:2 -o empty -e file:experr kyua report-html --action=514
 191 }
 192
 193
 194 utils_test_case force__yes
 195 force__yes_body() {
 196     run_tests "mock1"
 197
 198     atf_check -s exit:0 -o ignore -e empty kyua report-html
 199     test -f html/index.html || atf_fail "Expected file not created"
 200     rm html/index.html
 201     atf_check -s exit:0 -o ignore -e empty kyua report-html --force
 202     test -f html/index.html || atf_fail "Expected file not created"
 203 }
 204
 205
 206 utils_test_case force__no
 207 force__no_body() {
 208     run_tests "mock1"
 209
 210     atf_check -s exit:0 -o ignore -e empty kyua report-html
 211     test -f html/index.html || atf_fail "Expected file not created"
 212     rm html/index.html
 213
 214 cat >experr <<EOF
 215 kyua: E: Output directory 'html' already exists; maybe use --force?.
 216 EOF
 217     atf_check -s exit:2 -o empty -e file:experr kyua report-html
 218     test ! -f html/index.html || atf_fail "Not expected file created"
 219 }
 220
 221
 222 utils_test_case output__explicit
 223 output__explicit_body() {
 224     run_tests "mock1"
 225
 226     mkdir output
 227     atf_check -s exit:0 -o ignore -e empty kyua report-html --output=output/foo
 228     test ! -d html || atf_fail "Not expected directory created"
 229     test -f output/foo/index.html || atf_fail "Expected file not created"
 230 }
 231
 232
 233 utils_test_case results_filter__ok
 234 results_filter__ok_body() {
 235     utils_install_timestamp_wrapper
 236
 237     run_tests "mock1"
 238
 239     atf_check -s exit:0 -o ignore -e empty kyua report-html \
 240         --results-filter=passed
 241     for f in \
 242         html/index.html \
 243         html/context.html \
 244         html/simple_all_pass_pass.html \
 245         html/simple_some_fail_pass.html \
 246         html/metadata_no_properties.html \
 247         html/metadata_with_cleanup.html
 248     do
 249         test -f "${f}" || atf_fail "Missing ${f}"
 250     done
 251
 252     atf_check -o match:"2 TESTS FAILING" cat html/index.html
 253
 254     check_in_file html/simple_all_pass_pass.html \
 255         "This is the stdout of pass" "This is the stderr of pass"
 256     check_not_in_file html/simple_all_pass_pass.html \
 257         "This is the stdout of skip" "This is the stderr of skip" \
 258         "This is the stdout of fail" "This is the stderr of fail" \
 259         "Test case did not write anything to"
 260
 261     check_in_file html/simple_some_fail_pass.html \
 262         "Test case did not write anything to stdout" \
 263         "Test case did not write anything to stderr"
 264     check_not_in_file html/simple_some_fail_pass.html \
 265         "This is the stdout of pass" "This is the stderr of pass" \
 266         "This is the stdout of skip" "This is the stderr of skip" \
 267         "This is the stdout of fail" "This is the stderr of fail"
 268 }
 269
 270
 271 utils_test_case results_filter__invalid
 272 results_filter__invalid_body() {
 273     kyua db-exec "SELECT * FROM actions"
 274
 275     echo "kyua: E: Unknown result type 'foo-bar'." >experr
 276     atf_check -s exit:2 -o empty -e file:experr kyua report-html \
 277         --results-filter=passed,foo-bar
 278 }
 279
 280
 281 atf_init_test_cases() {
 282     atf_add_test_case default_behavior__ok
 283     atf_add_test_case default_behavior__no_actions
 284     atf_add_test_case default_behavior__no_store
 285
 286     atf_add_test_case action__explicit
 287     atf_add_test_case action__not_found
 288
 289     atf_add_test_case force__yes
 290     atf_add_test_case force__no
 291
 292     atf_add_test_case output__explicit
 293
 294     atf_add_test_case results_filter__ok
 295     atf_add_test_case results_filter__invalid
 296 }