external/bsd/atf/dist/atf-c++/tests.cpp

   1 //
   2 // Automated Testing Framework (atf)
   3 //
   4 // Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
   5 // All rights reserved.
   6 //
   7 // Redistribution and use in source and binary forms, with or without
   8 // modification, are permitted provided that the following conditions
   9 // are met:
  10 // 1. Redistributions of source code must retain the above copyright
  11 //    notice, this list of conditions and the following disclaimer.
  12 // 2. Redistributions in binary form must reproduce the above copyright
  13 //    notice, this list of conditions and the following disclaimer in the
  14 //    documentation and/or other materials provided with the distribution.
  15 //
  16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
  17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
  18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
  21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 //
  29
  30 extern "C" {
  31 #include <sys/types.h>
  32 #include <sys/stat.h>
  33 #include <sys/time.h>
  34 #include <sys/wait.h>
  35 #include <signal.h>
  36 #include <unistd.h>
  37 }
  38
  39 #include <algorithm>
  40 #include <cctype>
  41 #include <cerrno>
  42 #include <cstdlib>
  43 #include <cstring>
  44 #include <fstream>
  45 #include <iostream>
  46 #include <map>
  47 #include <memory>
  48 #include <sstream>
  49 #include <stdexcept>
  50 #include <vector>
  51
  52 extern "C" {
  53 #include "atf-c/error.h"
  54 #include "atf-c/object.h"
  55 }
  56
  57 #include "atf-c++/application.hpp"
  58 #include "atf-c++/config.hpp"
  59 #include "atf-c++/env.hpp"
  60 #include "atf-c++/exceptions.hpp"
  61 #include "atf-c++/expand.hpp"
  62 #include "atf-c++/formats.hpp"
  63 #include "atf-c++/fs.hpp"
  64 #include "atf-c++/io.hpp"
  65 #include "atf-c++/sanity.hpp"
  66 #include "atf-c++/signals.hpp"
  67 #include "atf-c++/tests.hpp"
  68 #include "atf-c++/text.hpp"
  69 #include "atf-c++/ui.hpp"
  70 #include "atf-c++/user.hpp"
  71
  72 namespace impl = atf::tests;
  73 #define IMPL_NAME "atf::tests"
  74
  75 // ------------------------------------------------------------------------
  76 // Auxiliary stuff for the timeout implementation.
  77 // ------------------------------------------------------------------------
  78
  79 namespace timeout {
  80     static pid_t current_body = 0;
  81     static bool killed = false;
  82
  83     void
  84     sigalrm_handler(int signo)
  85     {
  86         PRE(signo == SIGALRM);
  87
  88         if (current_body != 0) {
  89             ::killpg(current_body, SIGTERM);
  90             killed = true;
  91         }
  92     }
  93 } // namespace timeout
  94
  95 // ------------------------------------------------------------------------
  96 // The "tcr" class.
  97 // ------------------------------------------------------------------------
  98
  99 const impl::tcr::state impl::tcr::passed_state = atf_tcr_passed_state;
 100 const impl::tcr::state impl::tcr::failed_state = atf_tcr_failed_state;
 101 const impl::tcr::state impl::tcr::skipped_state = atf_tcr_skipped_state;
 102
 103 impl::tcr::tcr(state s)
 104 {
 105     PRE(s == passed_state);
 106
 107     atf_error_t err = atf_tcr_init(&m_tcr, s);
 108     if (atf_is_error(err))
 109         throw_atf_error(err);
 110 }
 111
 112 impl::tcr::tcr(state s, const std::string& r)
 113 {
 114     PRE(s == failed_state || s == skipped_state);
 115     PRE(!r.empty());
 116
 117     atf_error_t err = atf_tcr_init_reason_fmt(&m_tcr, s, "%s", r.c_str());
 118     if (atf_is_error(err))
 119         throw_atf_error(err);
 120 }
 121
 122 impl::tcr::tcr(const tcr& o)
 123 {
 124     if (o.get_state() == passed_state)
 125         atf_tcr_init(&m_tcr, o.get_state());
 126     else
 127         atf_tcr_init_reason_fmt(&m_tcr, o.get_state(), "%s",
 128                                 o.get_reason().c_str());
 129 }
 130
 131 impl::tcr::~tcr(void)
 132 {
 133     atf_tcr_fini(&m_tcr);
 134 }
 135
 136 impl::tcr::state
 137 impl::tcr::get_state(void)
 138     const
 139 {
 140     return atf_tcr_get_state(&m_tcr);
 141 }
 142
 143 const std::string
 144 impl::tcr::get_reason(void)
 145     const
 146 {
 147     const atf_dynstr_t* r = atf_tcr_get_reason(&m_tcr);
 148     return atf_dynstr_cstring(r);
 149 }
 150
 151 impl::tcr&
 152 impl::tcr::operator=(const tcr& o)
 153 {
 154     if (this != &o) {
 155         atf_tcr_fini(&m_tcr);
 156
 157         if (o.get_state() == passed_state)
 158             atf_tcr_init(&m_tcr, o.get_state());
 159         else
 160             atf_tcr_init_reason_fmt(&m_tcr, o.get_state(), "%s",
 161                                     o.get_reason().c_str());
 162     }
 163     return *this;
 164 }
 165
 166 // ------------------------------------------------------------------------
 167 // The "tc" class.
 168 // ------------------------------------------------------------------------
 169
 170 static std::map< atf_tc_t*, impl::tc* > wraps;
 171 static std::map< const atf_tc_t*, const impl::tc* > cwraps;
 172
 173 void
 174 impl::tc::wrap_head(atf_tc_t *tc)
 175 {
 176     std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
 177     INV(iter != wraps.end());
 178     (*iter).second->head();
 179 }
 180
 181 void
 182 impl::tc::wrap_body(const atf_tc_t *tc)
 183 {
 184     std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
 185         cwraps.find(tc);
 186     INV(iter != cwraps.end());
 187     (*iter).second->body();
 188 }
 189
 190 void
 191 impl::tc::wrap_cleanup(const atf_tc_t *tc)
 192 {
 193     std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
 194         cwraps.find(tc);
 195     INV(iter != cwraps.end());
 196     (*iter).second->cleanup();
 197 }
 198
 199 impl::tc::tc(const std::string& ident) :
 200     m_ident(ident)
 201 {
 202 }
 203
 204 impl::tc::~tc(void)
 205 {
 206     cwraps.erase(&m_tc);
 207     wraps.erase(&m_tc);
 208
 209     atf_tc_fini(&m_tc);
 210     atf_map_fini(&m_config);
 211 }
 212
 213 void
 214 impl::tc::init(const vars_map& config)
 215 {
 216     atf_error_t err;
 217
 218     err = atf_map_init(&m_config);
 219     if (atf_is_error(err))
 220         throw_atf_error(err);
 221
 222     for (vars_map::const_iterator iter = config.begin();
 223          iter != config.end(); iter++) {
 224         const char *var = (*iter).first.c_str();
 225         const char *val = (*iter).second.c_str();
 226
 227         err = atf_map_insert(&m_config, var, ::strdup(val), true);
 228         if (atf_is_error(err)) {
 229             atf_map_fini(&m_config);
 230             throw_atf_error(err);
 231         }
 232     }
 233
 234     wraps[&m_tc] = this;
 235     cwraps[&m_tc] = this;
 236
 237     err = atf_tc_init(&m_tc, m_ident.c_str(), wrap_head, wrap_body,
 238                       wrap_cleanup, &m_config);
 239     if (atf_is_error(err)) {
 240         atf_map_fini(&m_config);
 241         throw_atf_error(err);
 242     }
 243 }
 244
 245 bool
 246 impl::tc::has_config_var(const std::string& var)
 247     const
 248 {
 249     return atf_tc_has_config_var(&m_tc, var.c_str());
 250 }
 251
 252 bool
 253 impl::tc::has_md_var(const std::string& var)
 254     const
 255 {
 256     return atf_tc_has_md_var(&m_tc, var.c_str());
 257 }
 258
 259 const std::string
 260 impl::tc::get_config_var(const std::string& var)
 261     const
 262 {
 263     return atf_tc_get_config_var(&m_tc, var.c_str());
 264 }
 265
 266 const std::string
 267 impl::tc::get_config_var(const std::string& var, const std::string& defval)
 268     const
 269 {
 270     return atf_tc_get_config_var_wd(&m_tc, var.c_str(), defval.c_str());
 271 }
 272
 273 const std::string
 274 impl::tc::get_md_var(const std::string& var)
 275     const
 276 {
 277     return atf_tc_get_md_var(&m_tc, var.c_str());
 278 }
 279
 280 void
 281 impl::tc::set_md_var(const std::string& var, const std::string& val)
 282 {
 283     atf_error_t err = atf_tc_set_md_var(&m_tc, var.c_str(), val.c_str());
 284     if (atf_is_error(err))
 285         throw_atf_error(err);
 286 }
 287
 288 impl::tcr
 289 impl::tc::run(int fdout, int fderr, const fs::path& workdirbase)
 290     const
 291 {
 292     atf_tcr_t tcrc;
 293     tcr tcrr(tcr::failed_state, "UNINITIALIZED");
 294
 295     atf_error_t err = atf_tc_run(&m_tc, &tcrc, fdout, fderr,
 296                                  workdirbase.c_path());
 297     if (atf_is_error(err))
 298         throw_atf_error(err);
 299
 300     if (atf_tcr_has_reason(&tcrc)) {
 301         const atf_dynstr_t* r = atf_tcr_get_reason(&tcrc);
 302         tcrr = tcr(atf_tcr_get_state(&tcrc), atf_dynstr_cstring(r));
 303     } else {
 304         tcrr = tcr(atf_tcr_get_state(&tcrc));
 305     }
 306
 307     atf_tcr_fini(&tcrc);
 308     return tcrr;
 309 }
 310
 311 void
 312 impl::tc::cleanup(void)
 313     const
 314 {
 315 }
 316
 317 void
 318 impl::tc::require_prog(const std::string& prog)
 319     const
 320 {
 321     atf_tc_require_prog(prog.c_str());
 322 }
 323
 324 void
 325 impl::tc::pass(void)
 326 {
 327     atf_tc_pass();
 328 }
 329
 330 void
 331 impl::tc::fail(const std::string& reason)
 332 {
 333     atf_tc_fail("%s", reason.c_str());
 334 }
 335
 336 void
 337 impl::tc::skip(const std::string& reason)
 338 {
 339     atf_tc_skip("%s", reason.c_str());
 340 }
 341
 342 // ------------------------------------------------------------------------
 343 // The "tp" class.
 344 // ------------------------------------------------------------------------
 345
 346 class tp : public atf::application::app {
 347 public:
 348     typedef std::vector< impl::tc * > tc_vector;
 349
 350 private:
 351     static const char* m_description;
 352
 353     bool m_lflag;
 354     int m_results_fd;
 355     std::auto_ptr< std::ostream > m_results_os;
 356     atf::fs::path m_srcdir;
 357     atf::fs::path m_workdir;
 358     std::vector< std::string > m_tcnames;
 359
 360     atf::tests::vars_map m_vars;
 361
 362     std::string specific_args(void) const;
 363     options_set specific_options(void) const;
 364     void process_option(int, const char*);
 365
 366     void (*m_add_tcs)(tc_vector&);
 367     tc_vector m_tcs;
 368
 369     void parse_vflag(const std::string&);
 370     void handle_srcdir(void);
 371
 372     tc_vector init_tcs(void);
 373     static tc_vector filter_tcs(tc_vector,
 374                                 const std::vector< std::string >&);
 375
 376     std::ostream& results_stream(void);
 377
 378     int list_tcs(void);
 379     int run_tcs(void);
 380
 381 public:
 382     tp(void (*)(tc_vector&));
 383     ~tp(void);
 384
 385     int main(void);
 386 };
 387
 388 const char* tp::m_description =
 389     "This is an independent atf test program.";
 390
 391 tp::tp(void (*add_tcs)(tc_vector&)) :
 392     app(m_description, "atf-test-program(1)", "atf(7)"),
 393     m_lflag(false),
 394     m_results_fd(STDOUT_FILENO),
 395     m_srcdir("."),
 396     m_workdir(atf::config::get("atf_workdir")),
 397     m_add_tcs(add_tcs)
 398 {
 399 }
 400
 401 tp::~tp(void)
 402 {
 403     for (tc_vector::iterator iter = m_tcs.begin();
 404          iter != m_tcs.end(); iter++) {
 405         impl::tc* tc = *iter;
 406
 407         delete tc;
 408     }
 409 }
 410
 411 std::string
 412 tp::specific_args(void)
 413     const
 414 {
 415     return "[test_case1 [.. test_caseN]]";
 416 }
 417
 418 tp::options_set
 419 tp::specific_options(void)
 420     const
 421 {
 422     using atf::application::option;
 423     options_set opts;
 424     opts.insert(option('l', "", "List test cases and their purpose"));
 425     opts.insert(option('r', "fd", "The file descriptor to which the test "
 426                                   "program will send the results of the "
 427                                   "test cases"));
 428     opts.insert(option('s', "srcdir", "Directory where the test's data "
 429                                       "files are located"));
 430     opts.insert(option('v', "var=value", "Sets the configuration variable "
 431                                          "`var' to `value'"));
 432     opts.insert(option('w', "workdir", "Directory where the test's "
 433                                        "temporary files are located"));
 434     return opts;
 435 }
 436
 437 void
 438 tp::process_option(int ch, const char* arg)
 439 {
 440     switch (ch) {
 441     case 'l':
 442         m_lflag = true;
 443         break;
 444
 445     case 'r':
 446         {
 447             std::istringstream ss(arg);
 448             ss >> m_results_fd;
 449         }
 450         break;
 451
 452     case 's':
 453         m_srcdir = atf::fs::path(arg);
 454         break;
 455
 456     case 'v':
 457         parse_vflag(arg);
 458         break;
 459
 460     case 'w':
 461         m_workdir = atf::fs::path(arg);
 462         break;
 463
 464     default:
 465         UNREACHABLE;
 466     }
 467 }
 468
 469 void
 470 tp::parse_vflag(const std::string& str)
 471 {
 472     if (str.empty())
 473         throw std::runtime_error("-v requires a non-empty argument");
 474
 475     std::vector< std::string > ws = atf::text::split(str, "=");
 476     if (ws.size() == 1 && str[str.length() - 1] == '=') {
 477         m_vars[ws[0]] = "";
 478     } else {
 479         if (ws.size() != 2)
 480             throw std::runtime_error("-v requires an argument of the form "
 481                                      "var=value");
 482
 483         m_vars[ws[0]] = ws[1];
 484     }
 485 }
 486
 487 void
 488 tp::handle_srcdir(void)
 489 {
 490     if (!atf::fs::exists(m_srcdir / m_prog_name))
 491         throw std::runtime_error("Cannot find the test program in the "
 492                                  "source directory `" + m_srcdir.str() + "'");
 493
 494     if (!m_srcdir.is_absolute())
 495         m_srcdir = m_srcdir.to_absolute();
 496
 497     m_vars["srcdir"] = m_srcdir.str();
 498 }
 499
 500 tp::tc_vector
 501 tp::init_tcs(void)
 502 {
 503     m_add_tcs(m_tcs);
 504     for (tc_vector::iterator iter = m_tcs.begin();
 505          iter != m_tcs.end(); iter++) {
 506         impl::tc* tc = *iter;
 507
 508         tc->init(m_vars);
 509     }
 510     return m_tcs;
 511 }
 512
 513 //
 514 // An auxiliary unary predicate that compares the given test case's
 515 // identifier to the identifier stored in it.
 516 //
 517 class tc_equal_to_ident {
 518     const std::string& m_ident;
 519
 520 public:
 521     tc_equal_to_ident(const std::string& i) :
 522         m_ident(i)
 523     {
 524     }
 525
 526     bool operator()(const impl::tc* tc)
 527     {
 528         return tc->get_md_var("ident") == m_ident;
 529     }
 530 };
 531
 532 tp::tc_vector
 533 tp::filter_tcs(tc_vector tcs, const std::vector< std::string >& tcnames)
 534 {
 535     tc_vector tcso;
 536
 537     if (tcnames.empty()) {
 538         // Special case: added for efficiency because this is the most
 539         // typical situation.
 540         tcso = tcs;
 541     } else {
 542         // Collect all the test cases' identifiers.
 543         std::vector< std::string > ids;
 544         for (tc_vector::iterator iter = tcs.begin();
 545              iter != tcs.end(); iter++) {
 546             impl::tc* tc = *iter;
 547
 548             ids.push_back(tc->get_md_var("ident"));
 549         }
 550
 551         // Iterate over all names provided by the user and, for each one,
 552         // expand it as if it were a glob pattern.  Collect all expansions.
 553         std::vector< std::string > exps;
 554         for (std::vector< std::string >::const_iterator iter = tcnames.begin();
 555              iter != tcnames.end(); iter++) {
 556             const std::string& glob = *iter;
 557
 558             std::vector< std::string > ms =
 559                 atf::expand::expand_glob(glob, ids);
 560             if (ms.empty())
 561                 throw std::runtime_error("Unknown test case `" + glob + "'");
 562             exps.insert(exps.end(), ms.begin(), ms.end());
 563         }
 564
 565         // For each expansion, locate its corresponding test case and add
 566         // it to the output set.
 567         for (std::vector< std::string >::const_iterator iter = exps.begin();
 568              iter != exps.end(); iter++) {
 569             const std::string& name = *iter;
 570
 571             tc_vector::iterator tciter =
 572                 std::find_if(tcs.begin(), tcs.end(), tc_equal_to_ident(name));
 573             INV(tciter != tcs.end());
 574             tcso.push_back(*tciter);
 575         }
 576     }
 577
 578     return tcso;
 579 }
 580
 581 int
 582 tp::list_tcs(void)
 583 {
 584     tc_vector tcs = filter_tcs(init_tcs(), m_tcnames);
 585
 586     std::string::size_type maxlen = 0;
 587     for (tc_vector::const_iterator iter = tcs.begin();
 588          iter != tcs.end(); iter++) {
 589         const impl::tc* tc = *iter;
 590
 591         if (maxlen < tc->get_md_var("ident").length())
 592             maxlen = tc->get_md_var("ident").length();
 593     }
 594
 595     for (tc_vector::const_iterator iter = tcs.begin();
 596          iter != tcs.end(); iter++) {
 597         const impl::tc* tc = *iter;
 598
 599         std::cout << atf::ui::format_text_with_tag(tc->get_md_var("descr"),
 600                                                    tc->get_md_var("ident"),
 601                                                    false, maxlen + 4)
 602                   << std::endl;
 603     }
 604
 605     return EXIT_SUCCESS;
 606 }
 607
 608 std::ostream&
 609 tp::results_stream(void)
 610 {
 611     if (m_results_fd == STDOUT_FILENO)
 612         return std::cout;
 613     else if (m_results_fd == STDERR_FILENO)
 614         return std::cerr;
 615     else
 616         return *m_results_os;
 617 }
 618
 619 int
 620 tp::run_tcs(void)
 621 {
 622     tc_vector tcs = filter_tcs(init_tcs(), m_tcnames);
 623
 624     if (!atf::fs::exists(m_workdir))
 625         throw std::runtime_error("Cannot find the work directory `" +
 626                                  m_workdir.str() + "'");
 627
 628     int errcode = EXIT_SUCCESS;
 629
 630     atf::signals::signal_holder sighup(SIGHUP);
 631     atf::signals::signal_holder sigint(SIGINT);
 632     atf::signals::signal_holder sigterm(SIGTERM);
 633
 634     atf::formats::atf_tcs_writer w(results_stream(), std::cout, std::cerr,
 635                                    tcs.size());
 636     for (tc_vector::iterator iter = tcs.begin();
 637          iter != tcs.end(); iter++) {
 638         impl::tc* tc = *iter;
 639
 640         w.start_tc(tc->get_md_var("ident"));
 641         impl::tcr tcr = tc->run(STDOUT_FILENO, STDERR_FILENO, m_workdir);
 642         w.end_tc(tcr);
 643
 644         sighup.process();
 645         sigint.process();
 646         sigterm.process();
 647
 648         if (tcr.get_state() == impl::tcr::failed_state)
 649             errcode = EXIT_FAILURE;
 650     }
 651
 652     return errcode;
 653 }
 654
 655 int
 656 tp::main(void)
 657 {
 658     int errcode;
 659
 660     handle_srcdir();
 661
 662     for (int i = 0; i < m_argc; i++)
 663         m_tcnames.push_back(m_argv[i]);
 664
 665     if (m_lflag)
 666         errcode = list_tcs();
 667     else {
 668         if (m_results_fd != STDOUT_FILENO && m_results_fd != STDERR_FILENO) {
 669             atf::io::file_handle fh(m_results_fd);
 670             m_results_os =
 671                 std::auto_ptr< std::ostream >(new atf::io::postream(fh));
 672         }
 673         errcode = run_tcs();
 674     }
 675
 676     return errcode;
 677 }
 678
 679 namespace atf {
 680     namespace tests {
 681         int run_tp(int, char* const*, void (*)(tp::tc_vector&));
 682     }
 683 }
 684
 685 int
 686 impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&))
 687 {
 688     return tp(add_tcs).run(argc, argv);
 689 }