external/bsd/kyua-cli/dist/engine/testers_test.cpp

   1 // Copyright 2012 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "engine/testers.hpp"
  30
  31 extern "C" {
  32 #include <sys/stat.h>
  33
  34 #include <signal.h>
  35 }
  36
  37 #include <cstdlib>
  38
  39 #include <atf-c++.hpp>
  40
  41 #include "engine/exceptions.hpp"
  42 #include "utils/datetime.hpp"
  43 #include "utils/env.hpp"
  44 #include "utils/format/macros.hpp"
  45 #include "utils/fs/path.hpp"
  46 #include "utils/fs/operations.hpp"
  47 #include "utils/optional.ipp"
  48 #include "utils/passwd.hpp"
  49
  50 namespace datetime = utils::datetime;
  51 namespace fs = utils::fs;
  52 namespace passwd = utils::passwd;
  53
  54 using utils::none;
  55 using utils::optional;
  56
  57
  58 namespace {
  59
  60
  61 /// Creates a mock tester that exits cleanly.
  62 ///
  63 /// The interface accepted by the tester is 'mock'.  This tester outputs the
  64 /// arguments passed to it and then prints a message to both the stdout and the
  65 /// stderr.
  66 ///
  67 /// \param exit_status Code to exit with.
  68 static void
  69 create_mock_tester_exit(const int exit_status)
  70 {
  71     atf::utils::create_file(
  72         "kyua-mock-tester",
  73         F("#! /bin/sh\n"
  74           "while [ ${#} -gt 0 ]; do\n"
  75           "    echo \"Arg: ${1}\"\n"
  76           "    shift\n"
  77           "done\n"
  78           "echo 'tester output'\n"
  79           "echo 'tester error' 1>&2\n"
  80           "exit %s\n") % exit_status);
  81     ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1);
  82
  83     utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
  84 }
  85
  86
  87 /// Creates a mock tester that receives a signal.
  88 ///
  89 /// The interface accepted by the tester is 'mock'.  This tester outputs the
  90 /// arguments passed to it and then prints a message to both the stdout and the
  91 /// stderr.
  92 ///
  93 /// \param term_sig Signal to deliver to the tester.  If the tester does not
  94 ///     exit due to this reason, it exits with an arbitrary non-zero code.
  95 static void
  96 create_mock_tester_signal(const int term_sig)
  97 {
  98     atf::utils::create_file(
  99         "kyua-mock-tester",
 100         F("#! /bin/sh\n"
 101           "while [ ${#} -gt 0 ]; do\n"
 102           "    echo \"Arg: ${1}\"\n"
 103           "    shift\n"
 104           "done\n"
 105           "echo 'tester output'\n"
 106           "echo 'tester error' 1>&2\n"
 107           "kill -%s $$\n"
 108           "echo 'signal did not terminate the process\n"
 109           "exit 0\n") % term_sig);
 110     ATF_REQUIRE(::chmod("kyua-mock-tester", 0755) != -1);
 111
 112     utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
 113 }
 114
 115
 116 }  // anonymous namespace
 117
 118
 119 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__defaults);
 120 ATF_TEST_CASE_BODY(tester__list__defaults)
 121 {
 122     create_mock_tester_exit(EXIT_SUCCESS);
 123     engine::tester tester("mock", none, none);
 124     const std::string output = tester.list(fs::path("/foo/bar"));
 125
 126     const std::string exp_output =
 127         "Arg: list\n"
 128         "Arg: /foo/bar\n"
 129         "tester output\n"
 130         "tester error\n";
 131     ATF_REQUIRE_EQ(exp_output, output);
 132 }
 133
 134
 135 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__explicit_common_args);
 136 ATF_TEST_CASE_BODY(tester__list__explicit_common_args)
 137 {
 138     const passwd::user user("fake", 123, 456);
 139     const datetime::delta timeout(15, 0);
 140
 141     create_mock_tester_exit(EXIT_SUCCESS);
 142     engine::tester tester("mock", utils::make_optional(user),
 143                           utils::make_optional(timeout));
 144     const std::string output = tester.list(fs::path("/another/program/1"));
 145
 146     const std::string exp_output =
 147         "Arg: -u123\n"
 148         "Arg: -g456\n"
 149         "Arg: -t15\n"
 150         "Arg: list\n"
 151         "Arg: /another/program/1\n"
 152         "tester output\n"
 153         "tester error\n";
 154     ATF_REQUIRE_EQ(exp_output, output);
 155 }
 156
 157
 158 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__unknown_interface);
 159 ATF_TEST_CASE_BODY(tester__list__unknown_interface)
 160 {
 161     utils::setenv("KYUA_TESTERSDIR", ".");
 162     engine::tester tester("non-existent", none, none);
 163     ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent",
 164                          tester.list(fs::path("does-not-matter")));
 165 }
 166
 167
 168 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_fails);
 169 ATF_TEST_CASE_BODY(tester__list__tester_fails)
 170 {
 171     create_mock_tester_exit(EXIT_FAILURE);
 172     engine::tester tester("mock", none, none);
 173     ATF_REQUIRE_THROW_RE(
 174         engine::error,
 175         "Tester did not exit cleanly:.*tester output.*tester error",
 176         tester.list(fs::path("does-not-matter")));
 177 }
 178
 179
 180 ATF_TEST_CASE_WITHOUT_HEAD(tester__list__tester_crashes);
 181 ATF_TEST_CASE_BODY(tester__list__tester_crashes)
 182 {
 183     create_mock_tester_signal(SIGKILL);
 184     engine::tester tester("mock", none, none);
 185     ATF_REQUIRE_THROW_RE(
 186         engine::error,
 187         "Tester did not exit cleanly:.*tester output.*tester error",
 188         tester.list(fs::path("does-not-matter")));
 189 }
 190
 191
 192 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__defaults);
 193 ATF_TEST_CASE_BODY(tester__test__defaults)
 194 {
 195     std::map< std::string, std::string > vars;
 196
 197     create_mock_tester_exit(EXIT_FAILURE);
 198     engine::tester tester("mock", none, none);
 199     tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"),
 200                 fs::path("tester.out"), fs::path("tester.err"), vars);
 201
 202     const std::string exp_output =
 203         "Arg: test\n"
 204         "Arg: /foo/bar\n"
 205         "Arg: test-case\n"
 206         "Arg: /the/result/file\n"
 207         "tester output\n";
 208     const std::string exp_error =
 209         "tester error\n";
 210     ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output));
 211     ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error));
 212 }
 213
 214
 215 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__explicit_common_args_and_vars);
 216 ATF_TEST_CASE_BODY(tester__test__explicit_common_args_and_vars)
 217 {
 218     const passwd::user user("fake", 123, 456);
 219     const datetime::delta timeout(15, 0);
 220
 221     std::map< std::string, std::string > vars;
 222     vars["var1"] = "value1";
 223     vars["variable-2"] = "value with spaces";
 224
 225     create_mock_tester_exit(EXIT_SUCCESS);
 226     engine::tester tester("mock", utils::make_optional(user),
 227                           utils::make_optional(timeout));
 228     tester.test(fs::path("/foo/bar"), "test-case", fs::path("/the/result/file"),
 229                 fs::path("tester.out"), fs::path("tester.err"), vars);
 230
 231     const std::string exp_output =
 232         "Arg: -u123\n"
 233         "Arg: -g456\n"
 234         "Arg: -t15\n"
 235         "Arg: test\n"
 236         "Arg: -vvar1=value1\n"
 237         "Arg: -vvariable-2=value with spaces\n"
 238         "Arg: /foo/bar\n"
 239         "Arg: test-case\n"
 240         "Arg: /the/result/file\n"
 241         "tester output\n";
 242     const std::string exp_error =
 243         "tester error\n";
 244     ATF_REQUIRE(atf::utils::compare_file("tester.out", exp_output));
 245     ATF_REQUIRE(atf::utils::compare_file("tester.err", exp_error));
 246 }
 247
 248
 249 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__unknown_interface);
 250 ATF_TEST_CASE_BODY(tester__test__unknown_interface)
 251 {
 252     const std::map< std::string, std::string > vars;
 253
 254     utils::setenv("KYUA_TESTERSDIR", ".");
 255     engine::tester tester("non-existent", none, none);
 256     ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface non-existent",
 257                          tester.test(fs::path("foo"), "bar", fs::path("baz"),
 258                                      fs::path("out"), fs::path("err"), vars));
 259 }
 260
 261
 262 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_fails);
 263 ATF_TEST_CASE_BODY(tester__test__tester_fails)
 264 {
 265     const std::map< std::string, std::string > vars;
 266
 267     create_mock_tester_exit(2);
 268     engine::tester tester("mock", none, none);
 269     ATF_REQUIRE_THROW_RE(
 270         engine::error,
 271         "Tester failed with code 2; this is a bug",
 272         tester.test(fs::path("foo"), "bar", fs::path("baz"),
 273                     fs::path("out"), fs::path("err"), vars));
 274 }
 275
 276
 277 ATF_TEST_CASE_WITHOUT_HEAD(tester__test__tester_crashes);
 278 ATF_TEST_CASE_BODY(tester__test__tester_crashes)
 279 {
 280     const std::map< std::string, std::string > vars;
 281
 282     create_mock_tester_signal(SIGKILL);
 283     engine::tester tester("mock", none, none);
 284     ATF_REQUIRE_THROW_RE(
 285         engine::error,
 286         F("Tester received signal %s; this is a bug") % SIGKILL,
 287         tester.test(fs::path("foo"), "bar", fs::path("baz"),
 288                     fs::path("out"), fs::path("err"), vars));
 289 }
 290
 291
 292 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__default);
 293 ATF_TEST_CASE_BODY(tester_path__default)
 294 {
 295     ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("atf").str()));
 296     ATF_REQUIRE(atf::utils::file_exists(engine::tester_path("plain").str()));
 297 }
 298
 299
 300 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__custom);
 301 ATF_TEST_CASE_BODY(tester_path__custom)
 302 {
 303     fs::mkdir(fs::path("testers"), 0755);
 304     atf::utils::create_file("testers/kyua-mock-1-tester", "Not a binary");
 305     atf::utils::create_file("testers/kyua-mock-2-tester", "Not a binary");
 306     utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
 307
 308     const fs::path mock1 = engine::tester_path("mock-1");
 309     ATF_REQUIRE(mock1.is_absolute());
 310     ATF_REQUIRE(atf::utils::file_exists(mock1.str()));
 311
 312     const fs::path mock2 = engine::tester_path("mock-2");
 313     ATF_REQUIRE(mock2.is_absolute());
 314     ATF_REQUIRE(atf::utils::file_exists(mock2.str()));
 315
 316     ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface mock-3",
 317                          engine::tester_path("mock-3"));
 318 }
 319
 320
 321 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__cached);
 322 ATF_TEST_CASE_BODY(tester_path__cached)
 323 {
 324     fs::mkdir(fs::path("testers"), 0755);
 325     atf::utils::create_file("testers/kyua-mock-tester", "Not a binary");
 326     utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
 327
 328     const fs::path mock = engine::tester_path("mock");
 329     ATF_REQUIRE(atf::utils::file_exists(mock.str()));
 330     ATF_REQUIRE(::unlink(mock.c_str()) != -1);
 331     ATF_REQUIRE(!atf::utils::file_exists(mock.str()));
 332     ATF_REQUIRE_EQ(mock, engine::tester_path("mock"));
 333 }
 334
 335
 336 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__empty);
 337 ATF_TEST_CASE_BODY(tester_path__empty)
 338 {
 339     fs::mkdir(fs::path("testers"), 0755);
 340     atf::utils::create_file("testers/kyua--tester", "Not a binary");
 341     utils::setenv("KYUA_TESTERSDIR", (fs::current_path() / "testers").str());
 342
 343     ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface ",
 344                          engine::tester_path(""));
 345 }
 346
 347
 348 ATF_TEST_CASE_WITHOUT_HEAD(tester_path__missing);
 349 ATF_TEST_CASE_BODY(tester_path__missing)
 350 {
 351     utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
 352     ATF_REQUIRE_THROW_RE(engine::error, "Unknown interface plain",
 353                          engine::tester_path("plain"));
 354 }
 355
 356
 357 ATF_INIT_TEST_CASES(tcs)
 358 {
 359     ATF_ADD_TEST_CASE(tcs, tester__list__defaults);
 360     ATF_ADD_TEST_CASE(tcs, tester__list__explicit_common_args);
 361     ATF_ADD_TEST_CASE(tcs, tester__list__unknown_interface);
 362     ATF_ADD_TEST_CASE(tcs, tester__list__tester_fails);
 363     ATF_ADD_TEST_CASE(tcs, tester__list__tester_crashes);
 364
 365     ATF_ADD_TEST_CASE(tcs, tester__test__defaults);
 366     ATF_ADD_TEST_CASE(tcs, tester__test__explicit_common_args_and_vars);
 367     ATF_ADD_TEST_CASE(tcs, tester__test__unknown_interface);
 368     ATF_ADD_TEST_CASE(tcs, tester__test__tester_fails);
 369     ATF_ADD_TEST_CASE(tcs, tester__test__tester_crashes);
 370
 371     ATF_ADD_TEST_CASE(tcs, tester_path__default);
 372     ATF_ADD_TEST_CASE(tcs, tester_path__custom);
 373     ATF_ADD_TEST_CASE(tcs, tester_path__cached);
 374     ATF_ADD_TEST_CASE(tcs, tester_path__empty);
 375     ATF_ADD_TEST_CASE(tcs, tester_path__missing);
 376 }