external/bsd/atf/dist/tools/atf-check.cpp

   1 //
   2 // Automated Testing Framework (atf)
   3 //
   4 // Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
   5 // All rights reserved.
   6 //
   7 // Redistribution and use in source and binary forms, with or without
   8 // modification, are permitted provided that the following conditions
   9 // are met:
  10 // 1. Redistributions of source code must retain the above copyright
  11 //    notice, this list of conditions and the following disclaimer.
  12 // 2. Redistributions in binary form must reproduce the above copyright
  13 //    notice, this list of conditions and the following disclaimer in the
  14 //    documentation and/or other materials provided with the distribution.
  15 //
  16 // THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
  17 // CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
  18 // INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  19 // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20 // IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
  21 // DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22 // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  23 // GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25 // IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  26 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  27 // IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 //
  29
  30 extern "C" {
  31 #include <sys/types.h>
  32 #include <sys/wait.h>
  33 }
  34
  35 #include <cerrno>
  36 #include <cstdlib>
  37 #include <cstring>
  38 #include <fstream>
  39 #include <iostream>
  40 #include <iterator>
  41 #include <list>
  42 #include <utility>
  43
  44 #include "atf-c++/application.hpp"
  45 #include "atf-c++/exceptions.hpp"
  46 #include "atf-c++/check.hpp"
  47 #include "atf-c++/config.hpp"
  48 #include "atf-c++/fs.hpp"
  49 #include "atf-c++/io.hpp"
  50 #include "atf-c++/process.hpp"
  51 #include "atf-c++/sanity.hpp"
  52 #include "atf-c++/text.hpp"
  53 #include "atf-c++/utils.hpp"
  54
  55 // ------------------------------------------------------------------------
  56 // Auxiliary functions.
  57 // ------------------------------------------------------------------------
  58
  59 static
  60 std::string
  61 flatten_argv(char* const* argv)
  62 {
  63     std::string cmdline;
  64
  65     char* const* arg = &argv[0];
  66     while (*arg != NULL) {
  67         if (arg != &argv[0])
  68             cmdline += ' ';
  69
  70         cmdline += *arg;
  71
  72         arg++;
  73     }
  74
  75     return cmdline;
  76 }
  77
  78 static
  79 atf::check::check_result
  80 execute(const char* const* argv)
  81 {
  82     std::cout << "Executing command [ ";
  83     for (int i = 0; argv[i] != NULL; ++i)
  84         std::cout << argv[i] << " ";
  85     std::cout << "]" << std::endl;
  86
  87     atf::process::argv_array argva(argv);
  88     return atf::check::exec(argva);
  89 }
  90
  91 static
  92 atf::check::check_result
  93 execute_with_shell(char* const* argv)
  94 {
  95     const std::string cmd = flatten_argv(argv);
  96
  97     const char* sh_argv[4];
  98     sh_argv[0] = atf::config::get("atf_shell").c_str();
  99     sh_argv[1] = "-c";
 100     sh_argv[2] = cmd.c_str();
 101     sh_argv[3] = NULL;
 102     return execute(sh_argv);
 103 }
 104
 105 // ------------------------------------------------------------------------
 106 // The "atf_check" application.
 107 // ------------------------------------------------------------------------
 108
 109 class atf_check : public atf::application::app {
 110     enum output_check_t {
 111         oc_ignore,
 112         oc_inline,
 113         oc_file,
 114         oc_empty,
 115         oc_save
 116     };
 117
 118     enum status_check_t {
 119         sc_equal,
 120         sc_not_equal,
 121         sc_ignore
 122     };
 123
 124     bool m_xflag;
 125
 126     output_check_t m_stdout_check;
 127     std::string m_stdout_arg;
 128
 129     output_check_t m_stderr_check;
 130     std::string m_stderr_arg;
 131
 132     status_check_t m_status_check;
 133     int m_status_arg;
 134
 135     static const char* m_description;
 136
 137     bool file_empty(const atf::fs::path&) const;
 138     void print_diff(const atf::fs::path&, const atf::fs::path&) const;
 139     void print_file(const atf::fs::path&) const;
 140     std::string decode(const std::string&) const;
 141
 142     bool run_status_check(const atf::check::check_result&) const;
 143     bool run_output_check(const atf::check::check_result&,
 144                           const std::string&) const;
 145
 146     std::string specific_args(void) const;
 147     options_set specific_options(void) const;
 148     void process_option(int, const char*);
 149     void process_option_s(const std::string&);
 150     void process_option_o(const std::string&);
 151     void process_option_e(const std::string&);
 152
 153 public:
 154     atf_check(void);
 155     int main(void);
 156 };
 157
 158 const char* atf_check::m_description =
 159     "atf-check executes given command and analyzes its results.";
 160
 161 atf_check::atf_check(void) :
 162     app(m_description, "atf-check(1)", "atf(7)"),
 163     m_xflag(false),
 164     m_stdout_check(oc_empty),
 165     m_stderr_check(oc_empty),
 166     m_status_check(sc_equal),
 167     m_status_arg(0)
 168 {
 169 }
 170
 171 bool
 172 atf_check::file_empty(const atf::fs::path& p)
 173     const
 174 {
 175     atf::fs::file_info f(p);
 176
 177     return (f.get_size() == 0);
 178 }
 179
 180 void
 181 atf_check::print_diff(const atf::fs::path& p1, const atf::fs::path& p2)
 182     const
 183 {
 184     const atf::process::status s =
 185         atf::process::exec(atf::fs::path("diff"),
 186                            atf::process::argv_array("diff", "-u", p1.c_str(),
 187                                                     p2.c_str(), NULL),
 188                            atf::process::stream_connect(STDOUT_FILENO,
 189                                                         STDERR_FILENO),
 190                            atf::process::stream_inherit());
 191
 192     if (!s.exited())
 193         std::cerr << "Failed to run diff(3)" << std::endl;
 194
 195     if (s.exitstatus() != 1)
 196         std::cerr << "Error while running diff(3)" << std::endl;
 197 }
 198
 199 void
 200 atf_check::print_file(const atf::fs::path& p)
 201     const
 202 {
 203     std::ifstream f(p.c_str());
 204     f >> std::noskipws;
 205     std::istream_iterator< char > begin(f), end;
 206     std::ostream_iterator< char > out(std::cerr);
 207     std::copy(begin, end, out);
 208 }
 209
 210 std::string
 211 atf_check::decode(const std::string& s)
 212     const
 213 {
 214     size_t i;
 215     std::string res;
 216
 217     res.reserve(s.length());
 218
 219     i = 0;
 220     while (i < s.length()) {
 221         char c = s[i++];
 222
 223         if (c == '\\') {
 224             switch (s[i++]) {
 225             case 'a': c = '\a'; break;
 226             case 'b': c = '\b'; break;
 227             case 'c': break;
 228             case 'e': c = 033; break;
 229             case 'f': c = '\f'; break;
 230             case 'n': c = '\n'; break;
 231             case 'r': c = '\r'; break;
 232             case 't': c = '\t'; break;
 233             case 'v': c = '\v'; break;
 234             case '\\': break;
 235             case '0':
 236                 {
 237                     int count = 3;
 238                     c = 0;
 239                     while (--count >= 0 && (unsigned)(s[i] - '0') < 8)
 240                         c = (c << 3) + (s[i++] - '0');
 241                     break;
 242                 }
 243             default:
 244                 --i;
 245                 break;
 246             }
 247         }
 248
 249         res.push_back(c);
 250     }
 251
 252     return res;
 253 }
 254
 255 bool
 256 atf_check::run_status_check(const atf::check::check_result& r)
 257     const
 258 {
 259     bool retval = true;
 260
 261     if (!r.exited()) {
 262         std::cerr << "Fail: program did not exit cleanly" << std::endl;
 263         retval = false;
 264     } else {
 265         const int& status = r.exitcode();
 266
 267         if (m_status_check == sc_equal) {
 268             if (m_status_arg != status) {
 269                 std::cerr << "Fail: incorrect exit status: "
 270                           << status << ", expected: "
 271                           << m_status_arg << std::endl;
 272                 retval = false;
 273             }
 274         } else if (m_status_check == sc_not_equal) {
 275             if (m_status_arg == status) {
 276                 std::cerr << "Fail: incorrect exit status: "
 277                           << status << ", expected: "
 278                           << "anything other" << std::endl;
 279                 retval = false;
 280             }
 281         }
 282     }
 283
 284     if (retval == false) {
 285         std::cerr << "stdout:" << std::endl;
 286         print_file(r.stdout_path());
 287         std::cerr << std::endl;
 288
 289         std::cerr << "stderr:" << std::endl;
 290         print_file(r.stderr_path());
 291         std::cerr << std::endl;
 292     }
 293
 294     return retval;
 295 }
 296
 297 bool
 298 atf_check::run_output_check(const atf::check::check_result& r,
 299                             const std::string& stdxxx)
 300     const
 301 {
 302     atf::fs::path path("/");
 303     std::string arg;
 304     output_check_t check = m_stdout_check;
 305
 306     if (stdxxx == "stdout") {
 307         path = r.stdout_path();
 308         arg = m_stdout_arg.c_str();
 309         check = m_stdout_check;
 310     } else if (stdxxx == "stderr") {
 311         path = r.stderr_path();
 312         arg = m_stderr_arg.c_str();
 313         check = m_stderr_check;
 314     } else
 315         UNREACHABLE;
 316
 317     if (check == oc_empty) {
 318         if (!file_empty(path)) {
 319             std::cerr << "Fail: incorrect " << stdxxx << std::endl;
 320             print_diff(atf::fs::path("/dev/null"), path);
 321
 322             return false;
 323         }
 324     } else if (check == oc_file) {
 325         if (atf::io::cmp(path, atf::fs::path(arg)) != 0) {
 326             std::cerr << "Fail: incorrect " << stdxxx << std::endl;
 327             print_diff(atf::fs::path(arg), path);
 328
 329             return false;
 330         }
 331     } else if (check == oc_inline) {
 332         atf::fs::path path2 = atf::fs::path(atf::config::get("atf_workdir"))
 333                               / "inline.XXXXXX";
 334         atf::fs::temp_file temp(path2);
 335         temp << decode(arg);
 336         temp.close();
 337
 338         if (atf::io::cmp(path, temp.get_path()) != 0) {
 339             std::cerr << "Fail: incorrect " << stdxxx << std::endl;
 340             print_diff(temp.get_path(), path);
 341
 342             return false;
 343         }
 344     } else if (check == oc_save) {
 345         std::ifstream ifs(path.c_str(), std::fstream::binary);
 346         ifs >> std::noskipws;
 347         std::istream_iterator< char > begin(ifs), end;
 348
 349         std::ofstream ofs(arg.c_str(), std::fstream::binary
 350                                      | std::fstream::trunc);
 351         std::ostream_iterator <char> obegin(ofs);
 352
 353         std::copy(begin, end, obegin);
 354     }
 355
 356     return true;
 357 }
 358
 359 std::string
 360 atf_check::specific_args(void)
 361     const
 362 {
 363     return "<command>";
 364 }
 365
 366 atf_check::options_set
 367 atf_check::specific_options(void)
 368     const
 369 {
 370     using atf::application::option;
 371     options_set opts;
 372
 373     opts.insert(option('s', "qual:value", "Handle status. Qualifier "
 374                 "must be one of: ignore eq:<num> ne:<num>"));
 375     opts.insert(option('o', "action:arg", "Handle stdout. Action must be "
 376                 "one of: empty ignore file:<path> inline:<val> save:<path>"));
 377     opts.insert(option('e', "action:arg", "Handle stderr. Action must be "
 378                 "one of: empty ignore file:<path> inline:<val> save:<path>"));
 379     opts.insert(option('x', "", "Execute command as a shell command"));
 380
 381     return opts;
 382 }
 383
 384 void
 385 atf_check::process_option_s(const std::string& arg)
 386 {
 387     using atf::application::usage_error;
 388
 389     if (arg == "ignore") {
 390         m_status_check = sc_ignore;
 391         return;
 392     }
 393
 394     std::string::size_type pos = arg.find(':');
 395     std::string action = arg.substr(0, pos);
 396
 397     if (action == "eq")
 398         m_status_check = sc_equal;
 399     else if (action == "ne")
 400         m_status_check = sc_not_equal;
 401     else
 402         throw usage_error("Invalid value for -s option");
 403
 404     std::string value = arg.substr(pos + 1);
 405
 406     try {
 407         m_status_arg = atf::text::to_type< unsigned int >(value);
 408     } catch (std::runtime_error) {
 409         throw usage_error("Invalid value for -s option; must be an "
 410                           "integer in range 0-255");
 411     }
 412
 413     if ((m_status_arg < 0) || (m_status_arg > 255))
 414         throw usage_error("Invalid value for -s option; must be an "
 415                           "integer in range 0-255");
 416
 417 }
 418
 419 void
 420 atf_check::process_option_o(const std::string& arg)
 421 {
 422     using atf::application::usage_error;
 423
 424     std::string::size_type pos = arg.find(':');
 425     std::string action = arg.substr(0, pos);
 426
 427     if (action == "empty")
 428         m_stdout_check = oc_empty;
 429     else if (action == "ignore")
 430         m_stdout_check = oc_ignore;
 431     else if (action == "save")
 432         m_stdout_check = oc_save;
 433     else if (action == "inline")
 434         m_stdout_check = oc_inline;
 435     else if (action == "file")
 436         m_stdout_check = oc_file;
 437     else
 438         throw usage_error("Invalid value for -o option");
 439
 440     m_stdout_arg = arg.substr(pos + 1);
 441 }
 442
 443 void
 444 atf_check::process_option_e(const std::string& arg)
 445 {
 446     using atf::application::usage_error;
 447
 448     std::string::size_type pos = arg.find(':');
 449     std::string action = arg.substr(0, pos);
 450
 451     if (action == "empty")
 452         m_stderr_check = oc_empty;
 453     else if (action == "ignore")
 454         m_stderr_check = oc_ignore;
 455     else if (action == "save")
 456         m_stderr_check = oc_save;
 457     else if (action == "inline")
 458         m_stderr_check = oc_inline;
 459     else if (action == "file")
 460         m_stderr_check = oc_file;
 461     else
 462         throw usage_error("Invalid value for -e option");
 463
 464     m_stderr_arg = arg.substr(pos + 1);
 465 }
 466
 467 void
 468 atf_check::process_option(int ch, const char* arg)
 469 {
 470     switch (ch) {
 471     case 's':
 472         process_option_s(arg);
 473         break;
 474
 475     case 'o':
 476         process_option_o(arg);
 477         break;
 478
 479     case 'e':
 480         process_option_e(arg);
 481         break;
 482
 483     case 'x':
 484         m_xflag = true;
 485         break;
 486
 487     default:
 488         UNREACHABLE;
 489     }
 490 }
 491
 492 int
 493 atf_check::main(void)
 494 {
 495     if (m_argc < 1)
 496         throw atf::application::usage_error("No command specified");
 497
 498     int status = EXIT_FAILURE;
 499
 500     atf::check::check_result r =
 501         m_xflag ? execute_with_shell(m_argv) : execute(m_argv);
 502
 503     if ((run_status_check(r) == false) ||
 504         (run_output_check(r, "stderr") == false) ||
 505         (run_output_check(r, "stdout") == false))
 506         status = EXIT_FAILURE;
 507     else
 508         status = EXIT_SUCCESS;
 509
 510     return status;
 511 }
 512
 513 int
 514 main(int argc, char* const* argv)
 515 {
 516     return atf_check().run(argc, argv);
 517 }