external/bsd/atf/dist/tools/atf-run.cpp

   1 //
   2 // Automated Testing Framework (atf)
   3 //
   4 // Copyright (c) 2007 The NetBSD Foundation, Inc.
   5 // All rights reserved.
   6 //
   7 // Redistribution and use in source and binary forms, with or without
   8 // modification, are permitted provided that the following conditions
   9 // are met:
  10 // 1. Redistributions of source code must retain the above copyright
  11 //    notice, this list of conditions and the following disclaimer.
  12 // 2. Redistributions in binary form must reproduce the above copyright
  13 //    notice, this list of conditions and the following disclaimer in the
  14 //    documentation and/or other materials provided with the distribution.
  15 //
  16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
  17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
  18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
  21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 //
  29
  30 extern "C" {
  31 #include <sys/types.h>
  32 #include <sys/param.h>
  33 #include <sys/stat.h>
  34 #include <sys/wait.h>
  35 #include <unistd.h>
  36 }
  37
  38 #include <algorithm>
  39 #include <cassert>
  40 #include <cerrno>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <fstream>
  44 #include <iostream>
  45 #include <map>
  46 #include <string>
  47
  48 #include "application.hpp"
  49 #include "atffile.hpp"
  50 #include "config.hpp"
  51 #include "config_file.hpp"
  52 #include "env.hpp"
  53 #include "exceptions.hpp"
  54 #include "fs.hpp"
  55 #include "parser.hpp"
  56 #include "process.hpp"
  57 #include "requirements.hpp"
  58 #include "test-program.hpp"
  59 #include "text.hpp"
  60
  61 namespace {
  62
  63 typedef std::map< std::string, std::string > vars_map;
  64
  65 } // anonymous namespace
  66
  67 class atf_run : public tools::application::app {
  68     static const char* m_description;
  69
  70     vars_map m_cmdline_vars;
  71
  72     static vars_map::value_type parse_var(const std::string&);
  73
  74     void process_option(int, const char*);
  75     std::string specific_args(void) const;
  76     options_set specific_options(void) const;
  77
  78     void parse_vflag(const std::string&);
  79
  80     std::vector< std::string > conf_args(void) const;
  81
  82     size_t count_tps(std::vector< std::string >) const;
  83
  84     int run_test(const tools::fs::path&, tools::test_program::atf_tps_writer&,
  85                  const vars_map&);
  86     int run_test_directory(const tools::fs::path&,
  87                            tools::test_program::atf_tps_writer&);
  88     int run_test_program(const tools::fs::path&,
  89                          tools::test_program::atf_tps_writer&,
  90                          const vars_map&);
  91
  92     tools::test_program::test_case_result get_test_case_result(
  93         const std::string&, const tools::process::status&,
  94         const tools::fs::path&) const;
  95
  96 public:
  97     atf_run(void);
  98
  99     int main(void);
 100 };
 101
 102 static void
 103 sanitize_gdb_env(void)
 104 {
 105     try {
 106         tools::env::unset("TERM");
 107     } catch (...) {
 108         // Just swallow exceptions here; they cannot propagate into C, which
 109         // is where this function is called from, and even if these exceptions
 110         // appear they are benign.
 111     }
 112 }
 113
 114 static void
 115 dump_stacktrace(const tools::fs::path& tp, const tools::process::status& s,
 116                 const tools::fs::path& workdir,
 117                 tools::test_program::atf_tps_writer& w)
 118 {
 119     assert(s.signaled() && s.coredump());
 120
 121     w.stderr_tc("Test program crashed; attempting to get stack trace");
 122
 123     const tools::fs::path corename = workdir /
 124         (tp.leaf_name().substr(0, MAXCOMLEN) + ".core");
 125     if (!tools::fs::exists(corename)) {
 126         w.stderr_tc("Expected file " + corename.str() + " not found");
 127         return;
 128     }
 129
 130     const tools::fs::path gdb(GDB);
 131     const tools::fs::path gdbout = workdir / "gdb.out";
 132     const tools::process::argv_array args(gdb.leaf_name().c_str(), "-batch",
 133                                         "-q", "-ex", "bt", tp.c_str(),
 134                                         corename.c_str(), NULL);
 135     tools::process::status status = tools::process::exec(
 136         gdb, args,
 137         tools::process::stream_redirect_path(gdbout),
 138         tools::process::stream_redirect_path(tools::fs::path("/dev/null")),
 139         sanitize_gdb_env);
 140     if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) {
 141         w.stderr_tc("Execution of " GDB " failed");
 142         return;
 143     }
 144
 145     std::ifstream input(gdbout.c_str());
 146     if (input) {
 147         std::string line;
 148         while (std::getline(input, line).good())
 149             w.stderr_tc(line);
 150         input.close();
 151     }
 152
 153     w.stderr_tc("Stack trace complete");
 154 }
 155
 156 const char* atf_run::m_description =
 157     "atf-run is a tool that runs tests programs and collects their "
 158     "results.";
 159
 160 atf_run::atf_run(void) :
 161     app(m_description, "atf-run(1)", "atf(7)")
 162 {
 163 }
 164
 165 void
 166 atf_run::process_option(int ch, const char* arg)
 167 {
 168     switch (ch) {
 169     case 'v':
 170         parse_vflag(arg);
 171         break;
 172
 173     default:
 174         std::abort();
 175     }
 176 }
 177
 178 std::string
 179 atf_run::specific_args(void)
 180     const
 181 {
 182     return "[test-program1 .. test-programN]";
 183 }
 184
 185 atf_run::options_set
 186 atf_run::specific_options(void)
 187     const
 188 {
 189     using tools::application::option;
 190     options_set opts;
 191     opts.insert(option('v', "var=value", "Sets the configuration variable "
 192                                          "`var' to `value'; overrides "
 193                                          "values in configuration files"));
 194     return opts;
 195 }
 196
 197 void
 198 atf_run::parse_vflag(const std::string& str)
 199 {
 200     if (str.empty())
 201         throw std::runtime_error("-v requires a non-empty argument");
 202
 203     std::vector< std::string > ws = tools::text::split(str, "=");
 204     if (ws.size() == 1 && str[str.length() - 1] == '=') {
 205         m_cmdline_vars[ws[0]] = "";
 206     } else {
 207         if (ws.size() != 2)
 208             throw std::runtime_error("-v requires an argument of the form "
 209                                      "var=value");
 210
 211         m_cmdline_vars[ws[0]] = ws[1];
 212     }
 213 }
 214
 215 int
 216 atf_run::run_test(const tools::fs::path& tp,
 217                   tools::test_program::atf_tps_writer& w,
 218                   const vars_map& config)
 219 {
 220     tools::fs::file_info fi(tp);
 221
 222     int errcode;
 223     if (fi.get_type() == tools::fs::file_info::dir_type)
 224         errcode = run_test_directory(tp, w);
 225     else {
 226         const vars_map effective_config =
 227             tools::config_file::merge_configs(config, m_cmdline_vars);
 228
 229         errcode = run_test_program(tp, w, effective_config);
 230     }
 231     return errcode;
 232 }
 233
 234 int
 235 atf_run::run_test_directory(const tools::fs::path& tp,
 236                             tools::test_program::atf_tps_writer& w)
 237 {
 238     tools::atffile af = tools::read_atffile(tp / "Atffile");
 239
 240     vars_map test_suite_vars;
 241     {
 242         vars_map::const_iterator iter = af.props().find("test-suite");
 243         assert(iter != af.props().end());
 244         test_suite_vars = tools::config_file::read_config_files((*iter).second);
 245     }
 246
 247     bool ok = true;
 248     for (std::vector< std::string >::const_iterator iter = af.tps().begin();
 249          iter != af.tps().end(); iter++) {
 250         const bool result = run_test(tp / *iter, w,
 251             tools::config_file::merge_configs(af.conf(), test_suite_vars));
 252         ok &= (result == EXIT_SUCCESS);
 253     }
 254
 255     return ok ? EXIT_SUCCESS : EXIT_FAILURE;
 256 }
 257
 258 tools::test_program::test_case_result
 259 atf_run::get_test_case_result(const std::string& broken_reason,
 260                               const tools::process::status& s,
 261                               const tools::fs::path& resfile)
 262     const
 263 {
 264     using tools::text::to_string;
 265     using tools::test_program::read_test_case_result;
 266     using tools::test_program::test_case_result;
 267
 268     if (!broken_reason.empty()) {
 269         test_case_result tcr;
 270
 271         try {
 272             tcr = read_test_case_result(resfile);
 273
 274             if (tcr.state() == "expected_timeout") {
 275                 return tcr;
 276             } else {
 277                 return test_case_result("failed", -1, broken_reason);
 278             }
 279         } catch (const std::runtime_error&) {
 280             return test_case_result("failed", -1, broken_reason);
 281         }
 282     }
 283
 284     if (s.exited()) {
 285         test_case_result tcr;
 286
 287         try {
 288             tcr = read_test_case_result(resfile);
 289         } catch (const std::runtime_error& e) {
 290             return test_case_result("failed", -1, "Test case exited "
 291                 "normally but failed to create the results file: " +
 292                 std::string(e.what()));
 293         }
 294
 295         if (tcr.state() == "expected_death") {
 296             return tcr;
 297         } else if (tcr.state() == "expected_exit") {
 298             if (tcr.value() == -1 || s.exitstatus() == tcr.value())
 299                 return tcr;
 300             else
 301                 return test_case_result("failed", -1, "Test case was "
 302                     "expected to exit with a " + to_string(tcr.value()) +
 303                     " error code but returned " + to_string(s.exitstatus()));
 304         } else if (tcr.state() == "expected_failure") {
 305             if (s.exitstatus() == EXIT_SUCCESS)
 306                 return tcr;
 307             else
 308                 return test_case_result("failed", -1, "Test case returned an "
 309                     "error in expected_failure mode but it should not have");
 310         } else if (tcr.state() == "expected_signal") {
 311             return test_case_result("failed", -1, "Test case exited cleanly "
 312                 "but was expected to receive a signal");
 313         } else if (tcr.state() == "failed") {
 314             if (s.exitstatus() == EXIT_SUCCESS)
 315                 return test_case_result("failed", -1, "Test case "
 316                     "exited successfully but reported failure");
 317             else
 318                 return tcr;
 319         } else if (tcr.state() == "passed") {
 320             if (s.exitstatus() == EXIT_SUCCESS)
 321                 return tcr;
 322             else
 323                 return test_case_result("failed", -1, "Test case exited as "
 324                     "passed but reported an error");
 325         } else if (tcr.state() == "skipped") {
 326             if (s.exitstatus() == EXIT_SUCCESS)
 327                 return tcr;
 328             else
 329                 return test_case_result("failed", -1, "Test case exited as "
 330                     "skipped but reported an error");
 331         }
 332     } else if (s.signaled()) {
 333         test_case_result tcr;
 334
 335         try {
 336             tcr = read_test_case_result(resfile);
 337         } catch (const std::runtime_error&) {
 338             return test_case_result("failed", -1, "Test program received "
 339                 "signal " + tools::text::to_string(s.termsig()) +
 340                 (s.coredump() ? " (core dumped)" : ""));
 341         }
 342
 343         if (tcr.state() == "expected_death") {
 344             return tcr;
 345         } else if (tcr.state() == "expected_signal") {
 346             if (tcr.value() == -1 || s.termsig() == tcr.value())
 347                 return tcr;
 348             else
 349                 return test_case_result("failed", -1, "Test case was "
 350                     "expected to exit due to a " + to_string(tcr.value()) +
 351                     " signal but got " + to_string(s.termsig()));
 352         } else {
 353             return test_case_result("failed", -1, "Test program received "
 354                 "signal " + tools::text::to_string(s.termsig()) +
 355                 (s.coredump() ? " (core dumped)" : "") + " and created a "
 356                 "bogus results file");
 357         }
 358     }
 359     std::abort();
 360     return test_case_result();
 361 }
 362
 363 int
 364 atf_run::run_test_program(const tools::fs::path& tp,
 365                           tools::test_program::atf_tps_writer& w,
 366                           const vars_map& config)
 367 {
 368     int errcode = EXIT_SUCCESS;
 369
 370     tools::test_program::metadata md;
 371     try {
 372         md = tools::test_program::get_metadata(tp, config);
 373     } catch (const tools::parser::format_error& e) {
 374         w.start_tp(tp.str(), 0);
 375         w.end_tp("Invalid format for test case list: " + std::string(e.what()));
 376         return EXIT_FAILURE;
 377     } catch (const tools::parser::parse_errors& e) {
 378         const std::string reason = tools::text::join(e, "; ");
 379         w.start_tp(tp.str(), 0);
 380         w.end_tp("Invalid format for test case list: " + reason);
 381         return EXIT_FAILURE;
 382     }
 383
 384     tools::fs::temp_dir resdir(
 385         tools::fs::path(tools::config::get("atf_workdir")) / "atf-run.XXXXXX");
 386
 387     w.start_tp(tp.str(), md.test_cases.size());
 388     if (md.test_cases.empty()) {
 389         w.end_tp("Bogus test program: reported 0 test cases");
 390         errcode = EXIT_FAILURE;
 391     } else {
 392         for (std::map< std::string, vars_map >::const_iterator iter
 393              = md.test_cases.begin(); iter != md.test_cases.end(); iter++) {
 394             const std::string& tcname = (*iter).first;
 395             const vars_map& tcmd = (*iter).second;
 396
 397             w.start_tc(tcname);
 398
 399             try {
 400                 const std::string& reqfail = tools::check_requirements(
 401                     tcmd, config);
 402                 if (!reqfail.empty()) {
 403                     w.end_tc("skipped", reqfail);
 404                     continue;
 405                 }
 406             } catch (const std::runtime_error& e) {
 407                 w.end_tc("failed", e.what());
 408                 errcode = EXIT_FAILURE;
 409                 continue;
 410             }
 411
 412             const std::pair< int, int > user = tools::get_required_user(
 413                 tcmd, config);
 414
 415             tools::fs::path resfile = resdir.get_path() / "tcr";
 416             assert(!tools::fs::exists(resfile));
 417             try {
 418                 const bool has_cleanup = tools::text::to_bool(
 419                     (*tcmd.find("has.cleanup")).second);
 420
 421                 tools::fs::temp_dir workdir(tools::fs::path(tools::config::get(
 422                     "atf_workdir")) / "atf-run.XXXXXX");
 423                 if (user.first != -1 && user.second != -1) {
 424                     if (::chown(workdir.get_path().c_str(), user.first,
 425                                 user.second) == -1) {
 426                         throw tools::system_error("chown(" +
 427                             workdir.get_path().str() + ")", "chown(2) failed",
 428                             errno);
 429                     }
 430                     resfile = workdir.get_path() / "tcr";
 431                 }
 432
 433                 std::pair< std::string, const tools::process::status > s =
 434                     tools::test_program::run_test_case(
 435                         tp, tcname, "body", tcmd, config,
 436                         resfile, workdir.get_path(), w);
 437                 if (s.second.signaled() && s.second.coredump())
 438                     dump_stacktrace(tp, s.second, workdir.get_path(), w);
 439                 if (has_cleanup)
 440                     (void)tools::test_program::run_test_case(
 441                         tp, tcname, "cleanup", tcmd,
 442                         config, resfile, workdir.get_path(), w);
 443
 444                 // TODO: Force deletion of workdir.
 445
 446                 tools::test_program::test_case_result tcr =
 447                     get_test_case_result(s.first, s.second, resfile);
 448
 449                 w.end_tc(tcr.state(), tcr.reason());
 450                 if (tcr.state() == "failed")
 451                     errcode = EXIT_FAILURE;
 452             } catch (...) {
 453                 if (tools::fs::exists(resfile))
 454                     tools::fs::remove(resfile);
 455                 throw;
 456             }
 457             if (tools::fs::exists(resfile))
 458                 tools::fs::remove(resfile);
 459
 460         }
 461         w.end_tp("");
 462     }
 463
 464     return errcode;
 465 }
 466
 467 size_t
 468 atf_run::count_tps(std::vector< std::string > tps)
 469     const
 470 {
 471     size_t ntps = 0;
 472
 473     for (std::vector< std::string >::const_iterator iter = tps.begin();
 474          iter != tps.end(); iter++) {
 475         tools::fs::path tp(*iter);
 476         tools::fs::file_info fi(tp);
 477
 478         if (fi.get_type() == tools::fs::file_info::dir_type) {
 479             tools::atffile af = tools::read_atffile(tp / "Atffile");
 480             std::vector< std::string > aux = af.tps();
 481             for (std::vector< std::string >::iterator i2 = aux.begin();
 482                  i2 != aux.end(); i2++)
 483                 *i2 = (tp / *i2).str();
 484             ntps += count_tps(aux);
 485         } else
 486             ntps++;
 487     }
 488
 489     return ntps;
 490 }
 491
 492 static
 493 void
 494 call_hook(const std::string& tool, const std::string& hook)
 495 {
 496     const tools::fs::path sh(tools::config::get("atf_shell"));
 497     const tools::fs::path hooks =
 498         tools::fs::path(tools::config::get("atf_pkgdatadir")) / (tool + ".hooks");
 499
 500     const tools::process::status s =
 501         tools::process::exec(sh,
 502                            tools::process::argv_array(sh.c_str(), hooks.c_str(),
 503                                                     hook.c_str(), NULL),
 504                            tools::process::stream_inherit(),
 505                            tools::process::stream_inherit());
 506
 507
 508     if (!s.exited() || s.exitstatus() != EXIT_SUCCESS)
 509         throw std::runtime_error("Failed to run the '" + hook + "' hook "
 510                                  "for '" + tool + "'");
 511 }
 512
 513 int
 514 atf_run::main(void)
 515 {
 516     tools::atffile af = tools::read_atffile(tools::fs::path("Atffile"));
 517
 518     std::vector< std::string > tps;
 519     tps = af.tps();
 520     if (m_argc >= 1) {
 521         // TODO: Ensure that the given test names are listed in the
 522         // Atffile.  Take into account that the file can be using globs.
 523         tps.clear();
 524         for (int i = 0; i < m_argc; i++)
 525             tps.push_back(m_argv[i]);
 526     }
 527
 528     // Read configuration data for this test suite.
 529     vars_map test_suite_vars;
 530     {
 531         vars_map::const_iterator iter = af.props().find("test-suite");
 532         assert(iter != af.props().end());
 533         test_suite_vars = tools::config_file::read_config_files((*iter).second);
 534     }
 535
 536     tools::test_program::atf_tps_writer w(std::cout);
 537     call_hook("atf-run", "info_start_hook");
 538     w.ntps(count_tps(tps));
 539
 540     bool ok = true;
 541     for (std::vector< std::string >::const_iterator iter = tps.begin();
 542          iter != tps.end(); iter++) {
 543         const bool result = run_test(tools::fs::path(*iter), w,
 544             tools::config_file::merge_configs(af.conf(), test_suite_vars));
 545         ok &= (result == EXIT_SUCCESS);
 546     }
 547
 548     call_hook("atf-run", "info_end_hook");
 549
 550     return ok ? EXIT_SUCCESS : EXIT_FAILURE;
 551 }
 552
 553 int
 554 main(int argc, char* const* argv)
 555 {
 556     return atf_run().run(argc, argv);
 557 }