external/bsd/atf/dist/tools/atf-run.cpp

   1 //
   2 // Automated Testing Framework (atf)
   3 //
   4 // Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
   5 // All rights reserved.
   6 //
   7 // Redistribution and use in source and binary forms, with or without
   8 // modification, are permitted provided that the following conditions
   9 // are met:
  10 // 1. Redistributions of source code must retain the above copyright
  11 //    notice, this list of conditions and the following disclaimer.
  12 // 2. Redistributions in binary form must reproduce the above copyright
  13 //    notice, this list of conditions and the following disclaimer in the
  14 //    documentation and/or other materials provided with the distribution.
  15 //
  16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
  17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
  18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
  21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 //
  29
  30 #if defined(HAVE_CONFIG_H)
  31 #include "bconfig.h"
  32 #endif
  33
  34 extern "C" {
  35 #include <sys/types.h>
  36 #include <sys/wait.h>
  37 #include <unistd.h>
  38 }
  39
  40 #include <cerrno>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <fstream>
  44 #include <iostream>
  45 #include <map>
  46 #include <string>
  47
  48 #include "atf-c++/application.hpp"
  49 #include "atf-c++/atffile.hpp"
  50 #include "atf-c++/config.hpp"
  51 #include "atf-c++/env.hpp"
  52 #include "atf-c++/exceptions.hpp"
  53 #include "atf-c++/formats.hpp"
  54 #include "atf-c++/fs.hpp"
  55 #include "atf-c++/io.hpp"
  56 #include "atf-c++/parser.hpp"
  57 #include "atf-c++/process.hpp"
  58 #include "atf-c++/sanity.hpp"
  59 #include "atf-c++/tests.hpp"
  60 #include "atf-c++/text.hpp"
  61
  62 class config : public atf::formats::atf_config_reader {
  63     atf::tests::vars_map m_vars;
  64
  65     void
  66     got_var(const std::string& var, const std::string& name)
  67     {
  68         m_vars[var] = name;
  69     }
  70
  71 public:
  72     config(std::istream& is) :
  73         atf::formats::atf_config_reader(is)
  74     {
  75     }
  76
  77     const atf::tests::vars_map&
  78     get_vars(void)
  79         const
  80     {
  81         return m_vars;
  82     }
  83 };
  84
  85 class muxer : public atf::formats::atf_tcs_reader {
  86     atf::fs::path m_tp;
  87     atf::formats::atf_tps_writer m_writer;
  88
  89     bool m_inited, m_finalized;
  90     size_t m_ntcs;
  91     std::string m_tcname;
  92
  93     // Counters for the test cases run by the test program.
  94     size_t m_passed, m_failed, m_skipped;
  95
  96     void
  97     got_ntcs(size_t ntcs)
  98     {
  99         m_writer.start_tp(m_tp.str(), ntcs);
 100         m_inited = true;
 101         if (ntcs == 0)
 102             throw atf::formats::format_error("Bogus test program: reported "
 103                                              "0 test cases");
 104     }
 105
 106     void
 107     got_tc_start(const std::string& tcname)
 108     {
 109         m_tcname = tcname;
 110         m_writer.start_tc(tcname);
 111     }
 112
 113     void
 114     got_tc_end(const atf::tests::tcr& tcr)
 115     {
 116         const atf::tests::tcr::state& s = tcr.get_state();
 117         if (s == atf::tests::tcr::passed_state) {
 118             m_passed++;
 119         } else if (s == atf::tests::tcr::skipped_state) {
 120             m_skipped++;
 121         } else if (s == atf::tests::tcr::failed_state) {
 122             m_failed++;
 123         } else
 124             UNREACHABLE;
 125
 126         m_writer.end_tc(tcr);
 127         m_tcname = "";
 128     }
 129
 130     void
 131     got_stdout_line(const std::string& line)
 132     {
 133         m_writer.stdout_tc(line);
 134     }
 135
 136     void
 137     got_stderr_line(const std::string& line)
 138     {
 139         m_writer.stderr_tc(line);
 140     }
 141
 142 public:
 143     muxer(const atf::fs::path& tp, atf::formats::atf_tps_writer& w,
 144            atf::io::pistream& is) :
 145         atf::formats::atf_tcs_reader(is),
 146         m_tp(tp),
 147         m_writer(w),
 148         m_inited(false),
 149         m_finalized(false),
 150         m_passed(0),
 151         m_failed(0),
 152         m_skipped(0)
 153     {
 154     }
 155
 156     size_t
 157     failed(void)
 158         const
 159     {
 160         return m_failed;
 161     }
 162
 163     void
 164     finalize(const std::string& reason = "")
 165     {
 166         PRE(!m_finalized);
 167
 168         if (!m_inited)
 169             m_writer.start_tp(m_tp.str(), 0);
 170         if (!m_tcname.empty()) {
 171             INV(!reason.empty());
 172             got_tc_end(atf::tests::tcr(atf::tests::tcr::failed_state,
 173                                        "Bogus test program"));
 174         }
 175
 176         m_writer.end_tp(reason);
 177         m_finalized = true;
 178     }
 179
 180     ~muxer(void)
 181     {
 182         // The following is incorrect because we cannot throw an exception
 183         // from a destructor.  Let's just hope that this never happens.
 184         PRE(m_finalized);
 185     }
 186 };
 187
 188 template< class K, class V >
 189 void
 190 merge_maps(std::map< K, V >& dest, const std::map< K, V >& src)
 191 {
 192     for (typename std::map< K, V >::const_iterator iter = src.begin();
 193          iter != src.end(); iter++)
 194         dest[(*iter).first] = (*iter).second;
 195 }
 196
 197 class atf_run : public atf::application::app {
 198     static const char* m_description;
 199
 200     atf::tests::vars_map m_atffile_vars;
 201     atf::tests::vars_map m_cmdline_vars;
 202     atf::tests::vars_map m_config_vars;
 203
 204     static atf::tests::vars_map::value_type parse_var(const std::string&);
 205
 206     void process_option(int, const char*);
 207     std::string specific_args(void) const;
 208     options_set specific_options(void) const;
 209
 210     void parse_vflag(const std::string&);
 211
 212     void read_one_config(const atf::fs::path&);
 213     void read_config(const std::string&);
 214     std::vector< std::string > conf_args(void) const;
 215
 216     size_t count_tps(std::vector< std::string >) const;
 217
 218     int run_test(const atf::fs::path&,
 219                  atf::formats::atf_tps_writer&);
 220     int run_test_directory(const atf::fs::path&,
 221                            atf::formats::atf_tps_writer&);
 222     int run_test_program(const atf::fs::path&,
 223                          atf::formats::atf_tps_writer&);
 224
 225     struct test_data {
 226         const atf_run* m_this;
 227         const atf::fs::path& m_tp;
 228         atf::io::pipe& m_respipe;
 229
 230         test_data(const atf_run* t, const atf::fs::path& tp,
 231                   atf::io::pipe& respipe) :
 232             m_this(t),
 233             m_tp(tp),
 234             m_respipe(respipe)
 235         {
 236         }
 237     };
 238
 239     static void route_run_test_program_child(void *);
 240     void run_test_program_child(const atf::fs::path&,
 241                                 atf::io::pipe&) const;
 242     int run_test_program_parent(const atf::fs::path&,
 243                                 atf::formats::atf_tps_writer&,
 244                                 atf::process::child&,
 245                                 atf::io::pipe&);
 246
 247 public:
 248     atf_run(void);
 249
 250     int main(void);
 251 };
 252
 253 const char* atf_run::m_description =
 254     "atf-run is a tool that runs tests programs and collects their "
 255     "results.";
 256
 257 atf_run::atf_run(void) :
 258     app(m_description, "atf-run(1)", "atf(7)")
 259 {
 260 }
 261
 262 void
 263 atf_run::process_option(int ch, const char* arg)
 264 {
 265     switch (ch) {
 266     case 'v':
 267         parse_vflag(arg);
 268         break;
 269
 270     default:
 271         UNREACHABLE;
 272     }
 273 }
 274
 275 std::string
 276 atf_run::specific_args(void)
 277     const
 278 {
 279     return "[test-program1 .. test-programN]";
 280 }
 281
 282 atf_run::options_set
 283 atf_run::specific_options(void)
 284     const
 285 {
 286     using atf::application::option;
 287     options_set opts;
 288     opts.insert(option('v', "var=value", "Sets the configuration variable "
 289                                          "`var' to `value'; overrides "
 290                                          "values in configuration files"));
 291     return opts;
 292 }
 293
 294 void
 295 atf_run::parse_vflag(const std::string& str)
 296 {
 297     if (str.empty())
 298         throw std::runtime_error("-v requires a non-empty argument");
 299
 300     std::vector< std::string > ws = atf::text::split(str, "=");
 301     if (ws.size() == 1 && str[str.length() - 1] == '=') {
 302         m_cmdline_vars[ws[0]] = "";
 303     } else {
 304         if (ws.size() != 2)
 305             throw std::runtime_error("-v requires an argument of the form "
 306                                      "var=value");
 307
 308         m_cmdline_vars[ws[0]] = ws[1];
 309     }
 310 }
 311
 312 int
 313 atf_run::run_test(const atf::fs::path& tp,
 314                   atf::formats::atf_tps_writer& w)
 315 {
 316     atf::fs::file_info fi(tp);
 317
 318     int errcode;
 319     if (fi.get_type() == atf::fs::file_info::dir_type)
 320         errcode = run_test_directory(tp, w);
 321     else
 322         errcode = run_test_program(tp, w);
 323     return errcode;
 324 }
 325
 326 int
 327 atf_run::run_test_directory(const atf::fs::path& tp,
 328                             atf::formats::atf_tps_writer& w)
 329 {
 330     atf::atffile af(tp / "Atffile");
 331     m_atffile_vars = af.conf();
 332
 333     atf::tests::vars_map oldvars = m_config_vars;
 334     {
 335         atf::tests::vars_map::const_iterator iter =
 336             af.props().find("test-suite");
 337         INV(iter != af.props().end());
 338         read_config((*iter).second);
 339     }
 340
 341     bool ok = true;
 342     for (std::vector< std::string >::const_iterator iter = af.tps().begin();
 343          iter != af.tps().end(); iter++)
 344         ok &= (run_test(tp / *iter, w) == EXIT_SUCCESS);
 345
 346     m_config_vars = oldvars;
 347
 348     return ok ? EXIT_SUCCESS : EXIT_FAILURE;
 349 }
 350
 351 std::vector< std::string >
 352 atf_run::conf_args(void) const
 353 {
 354     using atf::tests::vars_map;
 355
 356     atf::tests::vars_map vars;
 357     std::vector< std::string > args;
 358
 359     merge_maps(vars, m_atffile_vars);
 360     merge_maps(vars, m_config_vars);
 361     merge_maps(vars, m_cmdline_vars);
 362
 363     for (vars_map::const_iterator i = vars.begin(); i != vars.end(); i++)
 364         args.push_back("-v" + (*i).first + "=" + (*i).second);
 365
 366     return args;
 367 }
 368
 369 void
 370 atf_run::route_run_test_program_child(void* v)
 371 {
 372     test_data* td = static_cast< test_data* >(v);
 373     td->m_this->run_test_program_child(td->m_tp, td->m_respipe);
 374     UNREACHABLE;
 375 }
 376
 377 void
 378 atf_run::run_test_program_child(const atf::fs::path& tp,
 379                                 atf::io::pipe& respipe)
 380     const
 381 {
 382     // Remap the results file descriptor to point to the parent too.
 383     // We use the 9th one (instead of a bigger one) because shell scripts
 384     // can only use the [0..9] file descriptors in their redirections.
 385     respipe.rend().close();
 386     respipe.wend().posix_remap(9);
 387
 388     // Prepare the test program's arguments.  We use dynamic memory and
 389     // do not care to release it.  We are going to die anyway very soon,
 390     // either due to exec(2) or to exit(3).
 391     std::vector< std::string > confargs = conf_args();
 392     char** args = new char*[4 + confargs.size()];
 393     {
 394         // 0: Program name.
 395         std::string progname = tp.leaf_name();
 396         args[0] = new char[progname.length() + 1];
 397         std::strcpy(args[0], progname.c_str());
 398
 399         // 1: The file descriptor to which the results will be printed.
 400         args[1] = new char[4];
 401         std::strcpy(args[1], "-r9");
 402
 403         // 2: The directory where the test program lives.
 404         atf::fs::path bp = tp.branch_path();
 405         if (!bp.is_absolute())
 406             bp = bp.to_absolute();
 407         const char* dir = bp.c_str();
 408         args[2] = new char[std::strlen(dir) + 3];
 409         std::strcpy(args[2], "-s");
 410         std::strcat(args[2], dir);
 411
 412         // [3..last - 1]: Configuration arguments.
 413         std::vector< std::string >::size_type i;
 414         for (i = 0; i < confargs.size(); i++) {
 415             const char* str = confargs[i].c_str();
 416             args[3 + i] = new char[std::strlen(str) + 1];
 417             std::strcpy(args[3 + i], str);
 418         }
 419
 420         // Last: Terminator.
 421         args[3 + i] = NULL;
 422     }
 423
 424     // Do the real exec and report any errors to the parent through the
 425     // only mechanism we can use: stderr.
 426     // TODO Try to make this fail.
 427     ::execv(tp.c_str(), args);
 428     std::cerr << "Failed to execute `" << tp.str() << "': "
 429               << std::strerror(errno) << std::endl;
 430     std::exit(EXIT_FAILURE);
 431 }
 432
 433 int
 434 atf_run::run_test_program_parent(const atf::fs::path& tp,
 435                                  atf::formats::atf_tps_writer& w,
 436                                  atf::process::child& c,
 437                                  atf::io::pipe& respipe)
 438 {
 439     // Get the input stream of stdout and stderr.
 440     atf::io::file_handle outfh = c.stdout_fd();
 441     atf::io::unbuffered_istream outin(outfh);
 442     atf::io::file_handle errfh = c.stderr_fd();
 443     atf::io::unbuffered_istream errin(errfh);
 444
 445     // Get the file descriptor and input stream of the results channel.
 446     respipe.wend().close();
 447     atf::io::pistream resin(respipe.rend());
 448
 449     // Process the test case's output and multiplex it into our output
 450     // stream as we read it.
 451     muxer m(tp, w, resin);
 452     std::string fmterr;
 453     try {
 454         m.read(outin, errin);
 455     } catch (const atf::parser::parse_errors& e) {
 456         fmterr = "There were errors parsing the output of the test "
 457                  "program:";
 458         for (atf::parser::parse_errors::const_iterator iter = e.begin();
 459              iter != e.end(); iter++) {
 460             fmterr += " Line " + atf::text::to_string((*iter).first) +
 461                       ": " + (*iter).second + ".";
 462         }
 463     } catch (const atf::formats::format_error& e) {
 464         fmterr = e.what();
 465     } catch (...) {
 466         UNREACHABLE;
 467     }
 468
 469     try {
 470         outin.close();
 471         errin.close();
 472         resin.close();
 473     } catch (...) {
 474         UNREACHABLE;
 475     }
 476
 477     const atf::process::status s = c.wait();
 478
 479     int code;
 480     if (s.exited()) {
 481         code = s.exitstatus();
 482         if (m.failed() > 0 && code == EXIT_SUCCESS) {
 483             code = EXIT_FAILURE;
 484             m.finalize("Test program returned success but some test "
 485                        "cases failed" +
 486                        (fmterr.empty() ? "" : (".  " + fmterr)));
 487         } else {
 488             code = fmterr.empty() ? code : EXIT_FAILURE;
 489             m.finalize(fmterr);
 490         }
 491     } else if (s.signaled()) {
 492         code = EXIT_FAILURE;
 493         m.finalize("Test program received signal " +
 494                    atf::text::to_string(s.termsig()) +
 495                    (s.coredump() ? " (core dumped)" : "") +
 496                    (fmterr.empty() ? "" : (".  " + fmterr)));
 497     } else
 498         throw std::runtime_error
 499             ("Child process " + atf::text::to_string(c.pid()) +
 500              " terminated with an unknown status condition");
 501     return code;
 502 }
 503
 504 int
 505 atf_run::run_test_program(const atf::fs::path& tp,
 506                           atf::formats::atf_tps_writer& w)
 507 {
 508     // XXX: This respipe is quite annoying.  The fact that we cannot
 509     // represent it as part of a portable fork API (which only supports
 510     // stdin, stdout and stderr) and not even in our own fork API means
 511     // that this will be a huge source of portability problems in the
 512     // future, should we ever want to port ATF to Win32.  I guess it'd
 513     // be worth revisiting the decision of using a third file descriptor
 514     // for results reporting sooner than later.  Alternative: use a
 515     // temporary file.
 516     atf::io::pipe respipe;
 517     test_data td(this, tp, respipe);
 518     atf::process::child c =
 519         atf::process::fork(route_run_test_program_child,
 520                            atf::process::stream_capture(),
 521                            atf::process::stream_capture(),
 522                            static_cast< void * >(&td));
 523
 524     return run_test_program_parent(tp, w, c, respipe);
 525 }
 526
 527 size_t
 528 atf_run::count_tps(std::vector< std::string > tps)
 529     const
 530 {
 531     size_t ntps = 0;
 532
 533     for (std::vector< std::string >::const_iterator iter = tps.begin();
 534          iter != tps.end(); iter++) {
 535         atf::fs::path tp(*iter);
 536         atf::fs::file_info fi(tp);
 537
 538         if (fi.get_type() == atf::fs::file_info::dir_type) {
 539             atf::atffile af = atf::atffile(tp / "Atffile");
 540             std::vector< std::string > aux = af.tps();
 541             for (std::vector< std::string >::iterator i2 = aux.begin();
 542                  i2 != aux.end(); i2++)
 543                 *i2 = (tp / *i2).str();
 544             ntps += count_tps(aux);
 545         } else
 546             ntps++;
 547     }
 548
 549     return ntps;
 550 }
 551
 552 void
 553 atf_run::read_one_config(const atf::fs::path& p)
 554 {
 555     std::ifstream is(p.c_str());
 556     if (is) {
 557         config reader(is);
 558         reader.read();
 559         merge_maps(m_config_vars, reader.get_vars());
 560     }
 561 }
 562
 563 void
 564 atf_run::read_config(const std::string& name)
 565 {
 566     std::vector< atf::fs::path > dirs;
 567     dirs.push_back(atf::fs::path(atf::config::get("atf_confdir")));
 568     if (atf::env::has("HOME"))
 569         dirs.push_back(atf::fs::path(atf::env::get("HOME")) / ".atf");
 570
 571     m_config_vars.clear();
 572     for (std::vector< atf::fs::path >::const_iterator iter = dirs.begin();
 573          iter != dirs.end(); iter++) {
 574         read_one_config((*iter) / "common.conf");
 575         read_one_config((*iter) / (name + ".conf"));
 576     }
 577 }
 578
 579 static
 580 void
 581 call_hook(const std::string& tool, const std::string& hook)
 582 {
 583     const atf::fs::path sh(atf::config::get("atf_shell"));
 584     const atf::fs::path hooks =
 585         atf::fs::path(atf::config::get("atf_pkgdatadir")) / (tool + ".hooks");
 586
 587     const atf::process::status s =
 588         atf::process::exec(sh,
 589                            atf::process::argv_array(sh.c_str(), hooks.c_str(),
 590                                                     hook.c_str(), NULL),
 591                            atf::process::stream_inherit(),
 592                            atf::process::stream_inherit());
 593
 594
 595     if (!s.exited() || s.exitstatus() != EXIT_SUCCESS)
 596         throw std::runtime_error("Failed to run the '" + hook + "' hook "
 597                                  "for '" + tool + "'");
 598 }
 599
 600 int
 601 atf_run::main(void)
 602 {
 603     atf::atffile af(atf::fs::path("Atffile"));
 604     m_atffile_vars = af.conf();
 605
 606     std::vector< std::string > tps;
 607     tps = af.tps();
 608     if (m_argc >= 1) {
 609         // TODO: Ensure that the given test names are listed in the
 610         // Atffile.  Take into account that the file can be using globs.
 611         tps.clear();
 612         for (int i = 0; i < m_argc; i++)
 613             tps.push_back(m_argv[i]);
 614     }
 615
 616     // Read configuration data for this test suite.
 617     {
 618         atf::tests::vars_map::const_iterator iter =
 619             af.props().find("test-suite");
 620         INV(iter != af.props().end());
 621         read_config((*iter).second);
 622     }
 623
 624     atf::formats::atf_tps_writer w(std::cout);
 625     call_hook("atf-run", "info_start_hook");
 626     w.ntps(count_tps(tps));
 627
 628     bool ok = true;
 629     for (std::vector< std::string >::const_iterator iter = tps.begin();
 630          iter != tps.end(); iter++)
 631         ok &= (run_test(atf::fs::path(*iter), w) == EXIT_SUCCESS);
 632
 633     call_hook("atf-run", "info_end_hook");
 634
 635     return ok ? EXIT_SUCCESS : EXIT_FAILURE;
 636 }
 637
 638 int
 639 main(int argc, char* const* argv)
 640 {
 641     return atf_run().run(argc, argv);
 642 }