external/bsd/kyua-cli/dist/cli/common_test.cpp

   1 // Copyright 2011 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "cli/common.hpp"
  30
  31 #include <fstream>
  32
  33 #include <atf-c++.hpp>
  34
  35 #include "engine/exceptions.hpp"
  36 #include "engine/filters.hpp"
  37 #include "engine/test_case.hpp"
  38 #include "engine/test_program.hpp"
  39 #include "engine/test_result.hpp"
  40 #include "utils/cmdline/exceptions.hpp"
  41 #include "utils/cmdline/globals.hpp"
  42 #include "utils/cmdline/parser.ipp"
  43 #include "utils/cmdline/ui_mock.hpp"
  44 #include "utils/datetime.hpp"
  45 #include "utils/env.hpp"
  46 #include "utils/fs/exceptions.hpp"
  47 #include "utils/fs/operations.hpp"
  48 #include "utils/fs/path.hpp"
  49 #include "utils/optional.ipp"
  50 #include "utils/sanity.hpp"
  51
  52 namespace cmdline = utils::cmdline;
  53 namespace config = utils::config;
  54 namespace datetime = utils::datetime;
  55 namespace fs = utils::fs;
  56
  57 using utils::optional;
  58
  59
  60 namespace {
  61
  62
  63 /// Syntactic sugar to instantiate engine::test_filter objects.
  64 inline engine::test_filter
  65 mkfilter(const char* test_program, const char* test_case)
  66 {
  67     return engine::test_filter(fs::path(test_program), test_case);
  68 }
  69
  70
  71 }  // anonymous namespace
  72
  73
  74 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__default);
  75 ATF_TEST_CASE_BODY(build_root_path__default)
  76 {
  77     std::map< std::string, std::vector< std::string > > options;
  78     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
  79
  80     ATF_REQUIRE(!cli::build_root_path(mock_cmdline));
  81 }
  82
  83
  84 ATF_TEST_CASE_WITHOUT_HEAD(build_root_path__explicit);
  85 ATF_TEST_CASE_BODY(build_root_path__explicit)
  86 {
  87     std::map< std::string, std::vector< std::string > > options;
  88     options["build-root"].push_back("/my//path");
  89     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
  90
  91     ATF_REQUIRE(cli::build_root_path(mock_cmdline));
  92     ATF_REQUIRE_EQ("/my/path", cli::build_root_path(mock_cmdline).get().str());
  93 }
  94
  95
  96 ATF_TEST_CASE_WITHOUT_HEAD(get_home__ok);
  97 ATF_TEST_CASE_BODY(get_home__ok)
  98 {
  99     const fs::path home("/foo/bar");
 100     utils::setenv("HOME", home.str());
 101     const optional< fs::path > computed = cli::get_home();
 102     ATF_REQUIRE(computed);
 103     ATF_REQUIRE_EQ(home, computed.get());
 104 }
 105
 106
 107 ATF_TEST_CASE_WITHOUT_HEAD(get_home__missing);
 108 ATF_TEST_CASE_BODY(get_home__missing)
 109 {
 110     utils::unsetenv("HOME");
 111     ATF_REQUIRE(!cli::get_home());
 112 }
 113
 114
 115 ATF_TEST_CASE_WITHOUT_HEAD(get_home__invalid);
 116 ATF_TEST_CASE_BODY(get_home__invalid)
 117 {
 118     utils::setenv("HOME", "");
 119     ATF_REQUIRE(!cli::get_home());
 120 }
 121
 122
 123 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__default);
 124 ATF_TEST_CASE_BODY(kyuafile_path__default)
 125 {
 126     std::map< std::string, std::vector< std::string > > options;
 127     options["kyuafile"].push_back(cli::kyuafile_option.default_value());
 128     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 129
 130     ATF_REQUIRE_EQ(cli::kyuafile_option.default_value(),
 131                    cli::kyuafile_path(mock_cmdline).str());
 132 }
 133
 134
 135 ATF_TEST_CASE_WITHOUT_HEAD(kyuafile_path__explicit);
 136 ATF_TEST_CASE_BODY(kyuafile_path__explicit)
 137 {
 138     std::map< std::string, std::vector< std::string > > options;
 139     options["kyuafile"].push_back("/my//path");
 140     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 141
 142     ATF_REQUIRE_EQ("/my/path", cli::kyuafile_path(mock_cmdline).str());
 143 }
 144
 145
 146 ATF_TEST_CASE_WITHOUT_HEAD(result_types__default);
 147 ATF_TEST_CASE_BODY(result_types__default)
 148 {
 149     std::map< std::string, std::vector< std::string > > options;
 150     options["results-filter"].push_back(
 151         cli::results_filter_option.default_value());
 152     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 153
 154     cli::result_types exp_types;
 155     exp_types.push_back(engine::test_result::skipped);
 156     exp_types.push_back(engine::test_result::expected_failure);
 157     exp_types.push_back(engine::test_result::broken);
 158     exp_types.push_back(engine::test_result::failed);
 159     ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
 160 }
 161
 162
 163 ATF_TEST_CASE_WITHOUT_HEAD(result_types__empty);
 164 ATF_TEST_CASE_BODY(result_types__empty)
 165 {
 166     std::map< std::string, std::vector< std::string > > options;
 167     options["results-filter"].push_back("");
 168     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 169
 170     cli::result_types exp_types;
 171     exp_types.push_back(engine::test_result::passed);
 172     exp_types.push_back(engine::test_result::skipped);
 173     exp_types.push_back(engine::test_result::expected_failure);
 174     exp_types.push_back(engine::test_result::broken);
 175     exp_types.push_back(engine::test_result::failed);
 176     ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
 177 }
 178
 179
 180 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__all);
 181 ATF_TEST_CASE_BODY(result_types__explicit__all)
 182 {
 183     std::map< std::string, std::vector< std::string > > options;
 184     options["results-filter"].push_back("passed,skipped,xfail,broken,failed");
 185     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 186
 187     cli::result_types exp_types;
 188     exp_types.push_back(engine::test_result::passed);
 189     exp_types.push_back(engine::test_result::skipped);
 190     exp_types.push_back(engine::test_result::expected_failure);
 191     exp_types.push_back(engine::test_result::broken);
 192     exp_types.push_back(engine::test_result::failed);
 193     ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
 194 }
 195
 196
 197 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__some);
 198 ATF_TEST_CASE_BODY(result_types__explicit__some)
 199 {
 200     std::map< std::string, std::vector< std::string > > options;
 201     options["results-filter"].push_back("skipped,broken");
 202     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 203
 204     cli::result_types exp_types;
 205     exp_types.push_back(engine::test_result::skipped);
 206     exp_types.push_back(engine::test_result::broken);
 207     ATF_REQUIRE(exp_types == cli::get_result_types(mock_cmdline));
 208 }
 209
 210
 211 ATF_TEST_CASE_WITHOUT_HEAD(result_types__explicit__invalid);
 212 ATF_TEST_CASE_BODY(result_types__explicit__invalid)
 213 {
 214     std::map< std::string, std::vector< std::string > > options;
 215     options["results-filter"].push_back("skipped,foo,broken");
 216     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 217
 218     ATF_REQUIRE_THROW_RE(std::runtime_error, "Unknown result type 'foo'",
 219                          cli::get_result_types(mock_cmdline));
 220 }
 221
 222
 223 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__create_directory__ok);
 224 ATF_TEST_CASE_BODY(store_path__default__create_directory__ok)
 225 {
 226     std::map< std::string, std::vector< std::string > > options;
 227     options["store"].push_back(cli::store_option.default_value());
 228     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 229
 230     const fs::path home("homedir");
 231     utils::setenv("HOME", home.str());
 232
 233     ATF_REQUIRE(!fs::exists(home / ".kyua"));
 234     ATF_REQUIRE_EQ(home / ".kyua/store.db", cli::store_path(mock_cmdline));
 235     ATF_REQUIRE(fs::exists(home / ".kyua"));
 236 }
 237
 238
 239 ATF_TEST_CASE(store_path__default__create_directory__fail);
 240 ATF_TEST_CASE_HEAD(store_path__default__create_directory__fail)
 241 {
 242     set_md_var("require.user", "unprivileged");
 243 }
 244 ATF_TEST_CASE_BODY(store_path__default__create_directory__fail)
 245 {
 246     std::map< std::string, std::vector< std::string > > options;
 247     options["store"].push_back(cli::store_option.default_value());
 248     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 249
 250     const fs::path home("homedir");
 251     utils::setenv("HOME", home.str());
 252     fs::mkdir(home, 0555);
 253
 254     ATF_REQUIRE_THROW(fs::error, cli::store_path(mock_cmdline).str());
 255     ATF_REQUIRE(!fs::exists(home / ".kyua"));
 256 }
 257
 258
 259 ATF_TEST_CASE_WITHOUT_HEAD(store_path__default__no_home);
 260 ATF_TEST_CASE_BODY(store_path__default__no_home)
 261 {
 262     std::map< std::string, std::vector< std::string > > options;
 263     options["store"].push_back(cli::store_option.default_value());
 264     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 265
 266     utils::unsetenv("HOME");
 267
 268     ATF_REQUIRE_EQ("kyua-store.db", cli::store_path(mock_cmdline).str());
 269 }
 270
 271
 272 ATF_TEST_CASE_WITHOUT_HEAD(store_path__explicit);
 273 ATF_TEST_CASE_BODY(store_path__explicit)
 274 {
 275     std::map< std::string, std::vector< std::string > > options;
 276     options["store"].push_back("/my//path");
 277     const cmdline::parsed_cmdline mock_cmdline(options, cmdline::args_vector());
 278
 279     const fs::path home("homedir");
 280     ATF_REQUIRE_EQ("/my/path", cli::store_path(mock_cmdline).str());
 281     ATF_REQUIRE(!fs::exists(home / ".kyua"));
 282 }
 283
 284
 285 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__none);
 286 ATF_TEST_CASE_BODY(parse_filters__none)
 287 {
 288     const cmdline::args_vector args;
 289     const std::set< engine::test_filter > filters = cli::parse_filters(args);
 290     ATF_REQUIRE(filters.empty());
 291 }
 292
 293
 294 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__ok);
 295 ATF_TEST_CASE_BODY(parse_filters__ok)
 296 {
 297     cmdline::args_vector args;
 298     args.push_back("foo");
 299     args.push_back("bar/baz");
 300     args.push_back("other:abc");
 301     args.push_back("other:bcd");
 302     const std::set< engine::test_filter > filters = cli::parse_filters(args);
 303
 304     std::set< engine::test_filter > exp_filters;
 305     exp_filters.insert(mkfilter("foo", ""));
 306     exp_filters.insert(mkfilter("bar/baz", ""));
 307     exp_filters.insert(mkfilter("other", "abc"));
 308     exp_filters.insert(mkfilter("other", "bcd"));
 309
 310     ATF_REQUIRE(exp_filters == filters);
 311 }
 312
 313
 314 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__duplicate);
 315 ATF_TEST_CASE_BODY(parse_filters__duplicate)
 316 {
 317     cmdline::args_vector args;
 318     args.push_back("foo/bar//baz");
 319     args.push_back("hello/world:yes");
 320     args.push_back("foo//bar/baz");
 321     ATF_REQUIRE_THROW_RE(cmdline::error, "Duplicate.*'foo/bar/baz'",
 322                          cli::parse_filters(args));
 323 }
 324
 325
 326 ATF_TEST_CASE_WITHOUT_HEAD(parse_filters__nondisjoint);
 327 ATF_TEST_CASE_BODY(parse_filters__nondisjoint)
 328 {
 329     cmdline::args_vector args;
 330     args.push_back("foo/bar");
 331     args.push_back("hello/world:yes");
 332     args.push_back("foo/bar:baz");
 333     ATF_REQUIRE_THROW_RE(cmdline::error, "'foo/bar'.*'foo/bar:baz'.*disjoint",
 334                          cli::parse_filters(args));
 335 }
 336
 337
 338 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__none);
 339 ATF_TEST_CASE_BODY(report_unused_filters__none)
 340 {
 341     std::set< engine::test_filter > unused;
 342
 343     cmdline::ui_mock ui;
 344     ATF_REQUIRE(!cli::report_unused_filters(unused, &ui));
 345     ATF_REQUIRE(ui.out_log().empty());
 346     ATF_REQUIRE(ui.err_log().empty());
 347 }
 348
 349
 350 ATF_TEST_CASE_WITHOUT_HEAD(report_unused_filters__some);
 351 ATF_TEST_CASE_BODY(report_unused_filters__some)
 352 {
 353     std::set< engine::test_filter > unused;
 354     unused.insert(mkfilter("a/b", ""));
 355     unused.insert(mkfilter("hey/d", "yes"));
 356
 357     cmdline::ui_mock ui;
 358     cmdline::init("progname");
 359     ATF_REQUIRE(cli::report_unused_filters(unused, &ui));
 360     ATF_REQUIRE(ui.out_log().empty());
 361     ATF_REQUIRE_EQ(2, ui.err_log().size());
 362     ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'a/b'",
 363                                              ui.err_log()));
 364     ATF_REQUIRE( atf::utils::grep_collection("No.*matched.*'hey/d:yes'",
 365                                              ui.err_log()));
 366 }
 367
 368
 369 ATF_TEST_CASE_WITHOUT_HEAD(format_delta);
 370 ATF_TEST_CASE_BODY(format_delta)
 371 {
 372     ATF_REQUIRE_EQ("0.000s", cli::format_delta(datetime::delta()));
 373     ATF_REQUIRE_EQ("0.012s", cli::format_delta(datetime::delta(0, 12300)));
 374     ATF_REQUIRE_EQ("0.999s", cli::format_delta(datetime::delta(0, 999000)));
 375     ATF_REQUIRE_EQ("51.321s", cli::format_delta(datetime::delta(51, 321000)));
 376 }
 377
 378
 379 ATF_TEST_CASE_WITHOUT_HEAD(format_result__no_reason);
 380 ATF_TEST_CASE_BODY(format_result__no_reason)
 381 {
 382     ATF_REQUIRE_EQ("passed", cli::format_result(
 383         engine::test_result(engine::test_result::passed)));
 384     ATF_REQUIRE_EQ("failed", cli::format_result(
 385         engine::test_result(engine::test_result::failed)));
 386 }
 387
 388
 389 ATF_TEST_CASE_WITHOUT_HEAD(format_result__with_reason);
 390 ATF_TEST_CASE_BODY(format_result__with_reason)
 391 {
 392     ATF_REQUIRE_EQ("broken: Something", cli::format_result(
 393         engine::test_result(engine::test_result::broken, "Something")));
 394     ATF_REQUIRE_EQ("expected_failure: A B C", cli::format_result(
 395         engine::test_result(engine::test_result::expected_failure, "A B C")));
 396     ATF_REQUIRE_EQ("failed: More text", cli::format_result(
 397         engine::test_result(engine::test_result::failed, "More text")));
 398     ATF_REQUIRE_EQ("skipped: Bye", cli::format_result(
 399         engine::test_result(engine::test_result::skipped, "Bye")));
 400 }
 401
 402
 403 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_case);
 404 ATF_TEST_CASE_BODY(format_test_case_id__test_case)
 405 {
 406     const engine::test_program test_program(
 407         "mock", fs::path("foo/bar/baz"), fs::path("unused-root"),
 408         "unused-suite-name", engine::metadata_builder().build());
 409     const engine::test_case test_case("mock", test_program, "abc",
 410                                       engine::metadata_builder().build());
 411     ATF_REQUIRE_EQ("foo/bar/baz:abc", cli::format_test_case_id(test_case));
 412 }
 413
 414
 415 ATF_TEST_CASE_WITHOUT_HEAD(format_test_case_id__test_filter);
 416 ATF_TEST_CASE_BODY(format_test_case_id__test_filter)
 417 {
 418     const engine::test_filter filter(fs::path("foo/bar"), "baz");
 419     ATF_REQUIRE_EQ("foo/bar:baz", cli::format_test_case_id(filter));
 420 }
 421
 422
 423 ATF_INIT_TEST_CASES(tcs)
 424 {
 425     ATF_ADD_TEST_CASE(tcs, build_root_path__default);
 426     ATF_ADD_TEST_CASE(tcs, build_root_path__explicit);
 427
 428     ATF_ADD_TEST_CASE(tcs, get_home__ok);
 429     ATF_ADD_TEST_CASE(tcs, get_home__missing);
 430     ATF_ADD_TEST_CASE(tcs, get_home__invalid);
 431
 432     ATF_ADD_TEST_CASE(tcs, kyuafile_path__default);
 433     ATF_ADD_TEST_CASE(tcs, kyuafile_path__explicit);
 434
 435     ATF_ADD_TEST_CASE(tcs, result_types__default);
 436     ATF_ADD_TEST_CASE(tcs, result_types__empty);
 437     ATF_ADD_TEST_CASE(tcs, result_types__explicit__all);
 438     ATF_ADD_TEST_CASE(tcs, result_types__explicit__some);
 439     ATF_ADD_TEST_CASE(tcs, result_types__explicit__invalid);
 440
 441     ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__ok);
 442     ATF_ADD_TEST_CASE(tcs, store_path__default__create_directory__fail);
 443     ATF_ADD_TEST_CASE(tcs, store_path__default__no_home);
 444     ATF_ADD_TEST_CASE(tcs, store_path__explicit);
 445
 446     ATF_ADD_TEST_CASE(tcs, parse_filters__none);
 447     ATF_ADD_TEST_CASE(tcs, parse_filters__ok);
 448     ATF_ADD_TEST_CASE(tcs, parse_filters__duplicate);
 449     ATF_ADD_TEST_CASE(tcs, parse_filters__nondisjoint);
 450
 451     ATF_ADD_TEST_CASE(tcs, report_unused_filters__none);
 452     ATF_ADD_TEST_CASE(tcs, report_unused_filters__some);
 453
 454     ATF_ADD_TEST_CASE(tcs, format_delta);
 455
 456     ATF_ADD_TEST_CASE(tcs, format_result__no_reason);
 457     ATF_ADD_TEST_CASE(tcs, format_result__with_reason);
 458
 459     ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_case);
 460     ATF_ADD_TEST_CASE(tcs, format_test_case_id__test_filter);
 461 }