external/bsd/kyua-cli/dist/engine/test_program_test.cpp

   1 // Copyright 2010 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "engine/test_program.hpp"
  30
  31 extern "C" {
  32 #include <sys/stat.h>
  33
  34 #include <signal.h>
  35 }
  36
  37 #include <sstream>
  38
  39 #include <atf-c++.hpp>
  40
  41 #include "engine/exceptions.hpp"
  42 #include "engine/test_result.hpp"
  43 #include "utils/env.hpp"
  44 #include "utils/format/macros.hpp"
  45 #include "utils/fs/operations.hpp"
  46 #include "utils/fs/path.hpp"
  47 #include "utils/optional.ipp"
  48
  49 namespace fs = utils::fs;
  50
  51
  52 namespace {
  53
  54
  55 /// Creates a mock tester that receives a signal.
  56 ///
  57 /// \param term_sig Signal to deliver to the tester.  If the tester does not
  58 ///     exit due to this reason, it exits with an arbitrary non-zero code.
  59 static void
  60 create_mock_tester_signal(const int term_sig)
  61 {
  62     const std::string tester_name = "kyua-mock-tester";
  63
  64     atf::utils::create_file(
  65         tester_name,
  66         F("#! /bin/sh\n"
  67           "kill -%s $$\n"
  68           "exit 0\n") % term_sig);
  69     ATF_REQUIRE(::chmod(tester_name.c_str(), 0755) != -1);
  70
  71     utils::setenv("KYUA_TESTERSDIR", fs::current_path().str());
  72 }
  73
  74
  75 }  // anonymous namespace
  76
  77
  78 ATF_TEST_CASE_WITHOUT_HEAD(ctor_and_getters);
  79 ATF_TEST_CASE_BODY(ctor_and_getters)
  80 {
  81     const engine::metadata md = engine::metadata_builder()
  82         .add_custom("foo", "bar")
  83         .build();
  84     const engine::test_program test_program(
  85         "mock", fs::path("binary"), fs::path("root"), "suite-name", md);
  86     ATF_REQUIRE_EQ("mock", test_program.interface_name());
  87     ATF_REQUIRE_EQ(fs::path("binary"), test_program.relative_path());
  88     ATF_REQUIRE_EQ(fs::current_path() / "root/binary",
  89                    test_program.absolute_path());
  90     ATF_REQUIRE_EQ(fs::path("root"), test_program.root());
  91     ATF_REQUIRE_EQ("suite-name", test_program.test_suite_name());
  92     ATF_REQUIRE_EQ(md, test_program.get_metadata());
  93 }
  94
  95
  96 ATF_TEST_CASE_WITHOUT_HEAD(find__ok);
  97 ATF_TEST_CASE_BODY(find__ok)
  98 {
  99     const engine::test_program test_program(
 100         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
 101         engine::metadata_builder().build());
 102     const engine::test_case_ptr test_case = test_program.find("main");
 103     ATF_REQUIRE_EQ(fs::path("non-existent"),
 104                    test_case->container_test_program().relative_path());
 105     ATF_REQUIRE_EQ("main", test_case->name());
 106 }
 107
 108
 109 ATF_TEST_CASE_WITHOUT_HEAD(find__missing);
 110 ATF_TEST_CASE_BODY(find__missing)
 111 {
 112     const engine::test_program test_program(
 113         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
 114         engine::metadata_builder().build());
 115     ATF_REQUIRE_THROW_RE(engine::not_found_error,
 116                          "case.*abc.*program.*non-existent",
 117                          test_program.find("abc"));
 118 }
 119
 120
 121 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__get);
 122 ATF_TEST_CASE_BODY(test_cases__get)
 123 {
 124     const engine::test_program test_program(
 125         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
 126         engine::metadata_builder().build());
 127     const engine::test_cases_vector& test_cases = test_program.test_cases();
 128     ATF_REQUIRE_EQ(1, test_cases.size());
 129     ATF_REQUIRE_EQ(fs::path("non-existent"),
 130                    test_cases[0]->container_test_program().relative_path());
 131     ATF_REQUIRE_EQ("main", test_cases[0]->name());
 132 }
 133
 134
 135 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__some);
 136 ATF_TEST_CASE_BODY(test_cases__some)
 137 {
 138     engine::test_program test_program(
 139         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
 140         engine::metadata_builder().build());
 141
 142     engine::test_cases_vector exp_test_cases;
 143     const engine::test_case test_case("plain", test_program, "main",
 144                                       engine::metadata_builder().build());
 145     exp_test_cases.push_back(engine::test_case_ptr(
 146         new engine::test_case(test_case)));
 147     test_program.set_test_cases(exp_test_cases);
 148
 149     ATF_REQUIRE_EQ(exp_test_cases, test_program.test_cases());
 150 }
 151
 152
 153 ATF_TEST_CASE_WITHOUT_HEAD(test_cases__tester_fails);
 154 ATF_TEST_CASE_BODY(test_cases__tester_fails)
 155 {
 156     engine::test_program test_program(
 157         "mock", fs::path("non-existent"), fs::path("."), "suite-name",
 158         engine::metadata_builder().build());
 159     create_mock_tester_signal(SIGSEGV);
 160
 161     const engine::test_cases_vector& test_cases = test_program.test_cases();
 162     ATF_REQUIRE_EQ(1, test_cases.size());
 163
 164     const engine::test_case_ptr& test_case = test_cases[0];
 165     ATF_REQUIRE_EQ("__test_cases_list__", test_case->name());
 166
 167     ATF_REQUIRE(test_case->fake_result());
 168     const engine::test_result result = test_case->fake_result().get();
 169     ATF_REQUIRE(engine::test_result::broken == result.type());
 170     ATF_REQUIRE_MATCH("Tester did not exit cleanly", result.reason());
 171 }
 172
 173
 174 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__copy);
 175 ATF_TEST_CASE_BODY(operators_eq_and_ne__copy)
 176 {
 177     const engine::test_program tp1(
 178         "plain", fs::path("non-existent"), fs::path("."), "suite-name",
 179         engine::metadata_builder().build());
 180     const engine::test_program tp2 = tp1;
 181     ATF_REQUIRE(  tp1 == tp2);
 182     ATF_REQUIRE(!(tp1 != tp2));
 183 }
 184
 185
 186 ATF_TEST_CASE_WITHOUT_HEAD(operators_eq_and_ne__not_copy);
 187 ATF_TEST_CASE_BODY(operators_eq_and_ne__not_copy)
 188 {
 189     const std::string base_interface("plain");
 190     const fs::path base_relative_path("the/test/program");
 191     const fs::path base_root("/the/root");
 192     const std::string base_test_suite("suite-name");
 193     const engine::metadata base_metadata = engine::metadata_builder()
 194         .add_custom("X-foo", "bar")
 195         .build();
 196
 197     engine::test_program base_tp(
 198         base_interface, base_relative_path, base_root, base_test_suite,
 199         base_metadata);
 200
 201     engine::test_cases_vector base_tcs;
 202     {
 203         const engine::test_case tc1("plain", base_tp, "main",
 204                                     engine::metadata_builder().build());
 205         base_tcs.push_back(engine::test_case_ptr(new engine::test_case(tc1)));
 206     }
 207     base_tp.set_test_cases(base_tcs);
 208
 209     // Construct with all same values.
 210     {
 211         engine::test_program other_tp(
 212             base_interface, base_relative_path, base_root, base_test_suite,
 213             base_metadata);
 214
 215         engine::test_cases_vector other_tcs;
 216         {
 217             const engine::test_case tc1("plain", other_tp, "main",
 218                                         engine::metadata_builder().build());
 219             other_tcs.push_back(engine::test_case_ptr(
 220                 new engine::test_case(tc1)));
 221         }
 222         other_tp.set_test_cases(other_tcs);
 223
 224         ATF_REQUIRE(  base_tp == other_tp);
 225         ATF_REQUIRE(!(base_tp != other_tp));
 226     }
 227
 228     // Different interface.
 229     {
 230         engine::test_program other_tp(
 231             "atf", base_relative_path, base_root, base_test_suite,
 232             base_metadata);
 233         other_tp.set_test_cases(base_tcs);
 234
 235         ATF_REQUIRE(!(base_tp == other_tp));
 236         ATF_REQUIRE(  base_tp != other_tp);
 237     }
 238
 239     // Different relative path.
 240     {
 241         engine::test_program other_tp(
 242             base_interface, fs::path("a/b/c"), base_root, base_test_suite,
 243             base_metadata);
 244         other_tp.set_test_cases(base_tcs);
 245
 246         ATF_REQUIRE(!(base_tp == other_tp));
 247         ATF_REQUIRE(  base_tp != other_tp);
 248     }
 249
 250     // Different root.
 251     {
 252         engine::test_program other_tp(
 253             base_interface, base_relative_path, fs::path("."), base_test_suite,
 254             base_metadata);
 255         other_tp.set_test_cases(base_tcs);
 256
 257         ATF_REQUIRE(!(base_tp == other_tp));
 258         ATF_REQUIRE(  base_tp != other_tp);
 259     }
 260
 261     // Different test suite.
 262     {
 263         engine::test_program other_tp(
 264             base_interface, base_relative_path, base_root, "different-suite",
 265             base_metadata);
 266         other_tp.set_test_cases(base_tcs);
 267
 268         ATF_REQUIRE(!(base_tp == other_tp));
 269         ATF_REQUIRE(  base_tp != other_tp);
 270     }
 271
 272     // Different metadata.
 273     {
 274         engine::test_program other_tp(
 275             base_interface, base_relative_path, base_root, base_test_suite,
 276             engine::metadata_builder().build());
 277         other_tp.set_test_cases(base_tcs);
 278
 279         ATF_REQUIRE(!(base_tp == other_tp));
 280         ATF_REQUIRE(  base_tp != other_tp);
 281     }
 282
 283     // Different test cases.
 284     {
 285         engine::test_program other_tp(
 286             base_interface, base_relative_path, base_root, base_test_suite,
 287             base_metadata);
 288
 289         engine::test_cases_vector other_tcs;
 290         {
 291             const engine::test_case tc1("atf", base_tp, "foo",
 292                                         engine::metadata_builder().build());
 293             other_tcs.push_back(engine::test_case_ptr(
 294                                     new engine::test_case(tc1)));
 295         }
 296         other_tp.set_test_cases(other_tcs);
 297
 298         ATF_REQUIRE(!(base_tp == other_tp));
 299         ATF_REQUIRE(  base_tp != other_tp);
 300     }
 301 }
 302
 303
 304 ATF_TEST_CASE_WITHOUT_HEAD(output__no_test_cases);
 305 ATF_TEST_CASE_BODY(output__no_test_cases)
 306 {
 307     engine::test_program tp(
 308         "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
 309         engine::metadata_builder().add_allowed_architecture("a").build());
 310     tp.set_test_cases(engine::test_cases_vector());
 311
 312     std::ostringstream str;
 313     str << tp;
 314     ATF_REQUIRE_EQ(
 315         "test_program{interface='plain', binary='binary/path', "
 316         "root='/the/root', test_suite='suite-name', "
 317         "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
 318         "description='', has_cleanup='false', "
 319         "required_configs='', required_files='', required_memory='0', "
 320         "required_programs='', required_user='', timeout='300'}, "
 321         "test_cases=[]}",
 322         str.str());
 323 }
 324
 325
 326 ATF_TEST_CASE_WITHOUT_HEAD(output__some_test_cases);
 327 ATF_TEST_CASE_BODY(output__some_test_cases)
 328 {
 329     engine::test_program tp(
 330         "plain", fs::path("binary/path"), fs::path("/the/root"), "suite-name",
 331         engine::metadata_builder().add_allowed_architecture("a").build());
 332
 333     const engine::test_case_ptr tc1(new engine::test_case(
 334         "plain", tp, "the-name", engine::metadata_builder()
 335         .add_allowed_platform("foo").add_custom("X-bar", "baz").build()));
 336     const engine::test_case_ptr tc2(new engine::test_case(
 337         "plain", tp, "another-name", engine::metadata_builder().build()));
 338     engine::test_cases_vector tcs;
 339     tcs.push_back(tc1);
 340     tcs.push_back(tc2);
 341     tp.set_test_cases(tcs);
 342
 343     std::ostringstream str;
 344     str << tp;
 345     ATF_REQUIRE_EQ(
 346         "test_program{interface='plain', binary='binary/path', "
 347         "root='/the/root', test_suite='suite-name', "
 348         "metadata=metadata{allowed_architectures='a', allowed_platforms='', "
 349         "description='', has_cleanup='false', "
 350         "required_configs='', required_files='', required_memory='0', "
 351         "required_programs='', required_user='', timeout='300'}, "
 352         "test_cases=["
 353         "test_case{interface='plain', name='the-name', "
 354         "metadata=metadata{allowed_architectures='', allowed_platforms='foo', "
 355         "custom.X-bar='baz', description='', has_cleanup='false', "
 356         "required_configs='', required_files='', required_memory='0', "
 357         "required_programs='', required_user='', timeout='300'}}, "
 358         "test_case{interface='plain', name='another-name', "
 359         "metadata=metadata{allowed_architectures='', allowed_platforms='', "
 360         "description='', has_cleanup='false', "
 361         "required_configs='', required_files='', required_memory='0', "
 362         "required_programs='', required_user='', timeout='300'}}]}",
 363         str.str());
 364 }
 365
 366
 367 ATF_INIT_TEST_CASES(tcs)
 368 {
 369     // TODO(jmmv): These tests have ceased to be realistic with the move to
 370     // TestersDesign.  We probably should have some (few!) integration tests for
 371     // the various known testers... or, alternatively, provide a mock tester to
 372     // run our tests with.
 373     ATF_ADD_TEST_CASE(tcs, ctor_and_getters);
 374     ATF_ADD_TEST_CASE(tcs, find__ok);
 375     ATF_ADD_TEST_CASE(tcs, find__missing);
 376     ATF_ADD_TEST_CASE(tcs, test_cases__get);
 377     ATF_ADD_TEST_CASE(tcs, test_cases__some);
 378     ATF_ADD_TEST_CASE(tcs, test_cases__tester_fails);
 379
 380     ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__copy);
 381     ATF_ADD_TEST_CASE(tcs, operators_eq_and_ne__not_copy);
 382
 383     ATF_ADD_TEST_CASE(tcs, output__no_test_cases);
 384     ATF_ADD_TEST_CASE(tcs, output__some_test_cases);
 385 }