external/bsd/kyua-cli/dist/cli/cmd_report_tap.cpp

   1 // Copyright 2011 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "cli/cmd_report.hpp"
  30 #include "cli/cmd_report_tap.hpp"
  31
  32 #include <cstddef>
  33 #include <cstdlib>
  34 #include <fstream>
  35 #include <map>
  36 #include <vector>
  37
  38 #include "cli/common.ipp"
  39 #include "engine/action.hpp"
  40 #include "engine/context.hpp"
  41 #include "engine/drivers/scan_action.hpp"
  42 #include "engine/test_result.hpp"
  43 #include "utils/cmdline/exceptions.hpp"
  44 #include "utils/cmdline/parser.ipp"
  45 #include "utils/defs.hpp"
  46 #include "utils/format/macros.hpp"
  47 #include "utils/optional.ipp"
  48
  49 namespace cmdline = utils::cmdline;
  50 namespace config = utils::config;
  51 namespace datetime = utils::datetime;
  52 namespace fs = utils::fs;
  53 namespace scan_action = engine::drivers::scan_action;
  54
  55 using cli::cmd_report_tap;
  56 using utils::optional;
  57
  58
  59 namespace {
  60
  61
  62 /// Generates a plain-text report intended to be printed to the console.
  63 class console_tap_hooks : public scan_action::base_hooks {
  64     /// Indirection to print the output to the correct file stream.
  65     cli::file_writer _writer;
  66
  67     /// Whether to include the runtime context in the output or not.
  68     const bool _show_context;
  69
  70     /// Collection of result types to include in the report.
  71     const cli::result_types& _results_filters;
  72
  73     /// The action ID loaded.
  74     int64_t _action_id;
  75
  76     /// The total run time of the tests.
  77     datetime::delta _runtime;
  78
  79     /// Representation of a single result.
  80     struct result_data {
  81         /// The relative path to the test program.
  82         fs::path binary_path;
  83
  84         /// The name of the test case.
  85         std::string test_case_name;
  86
  87         /// The result of the test case.
  88         engine::test_result result;
  89
  90         /// The duration of the test case execution.
  91         datetime::delta duration;
  92
  93         /// Constructs a new results data.
  94         ///
  95         /// \param binary_path_ The relative path to the test program.
  96         /// \param test_case_name_ The name of the test case.
  97         /// \param result_ The result of the test case.
  98         /// \param duration_ The duration of the test case execution.
  99         result_data(const fs::path& binary_path_,
 100                     const std::string& test_case_name_,
 101                     const engine::test_result& result_,
 102                     const datetime::delta& duration_) :
 103             binary_path(binary_path_), test_case_name(test_case_name_),
 104             result(result_), duration(duration_)
 105         {
 106         }
 107     };
 108
 109     /// Results received, broken down by their type.
 110     ///
 111     /// Note that this may not include all results, as keeping the whole list in
 112     /// memory may be too much.
 113     std::map< engine::test_result::result_type,
 114               std::vector< result_data > > _results;
 115
 116     /// Prints the execution context to the output.
 117     ///
 118     /// \param context The context to dump.
 119     void
 120     print_context(const engine::context& context)
 121     {
 122         _writer("  ===> Execution context");
 123
 124         _writer(F("  Current directory: %s") % context.cwd());
 125         const std::map< std::string, std::string >& env = context.env();
 126         if (env.empty())
 127             _writer("  No environment variables recorded");
 128         else {
 129             _writer("  Environment variables:");
 130             for (std::map< std::string, std::string >::const_iterator
 131                      iter = env.begin(); iter != env.end(); iter++) {
 132                 _writer(F("    %s=%s") % (*iter).first % (*iter).second);
 133             }
 134         }
 135     }
 136
 137     /// Counts how many results of a given type have been received.
 138     std::size_t
 139     count_results(const engine::test_result::result_type type)
 140     {
 141         const std::map< engine::test_result::result_type,
 142                         std::vector< result_data > >::const_iterator iter =
 143             _results.find(type);
 144         if (iter == _results.end())
 145             return 0;
 146         else
 147             return (*iter).second.size();
 148     }
 149
 150     /// Prints a set of results.
 151     void
 152     print_results(const engine::test_result::result_type type,
 153                   const char* title)
 154     {
 155         const std::map< engine::test_result::result_type,
 156                         std::vector< result_data > >::const_iterator iter2 =
 157             _results.find(type);
 158         if (iter2 == _results.end())
 159             return;
 160         const std::vector< result_data >& all = (*iter2).second;
 161
 162         std::string result = "";
 163         std::string comment = "";
 164
 165         if (type == engine::test_result::failed)
 166            result = "not ";
 167
 168         if (type == engine::test_result::broken) {
 169            result = "not ";
 170            comment = " # broken";
 171         }
 172
 173         if (type == engine::test_result::skipped)
 174            comment = " # skip";
 175
 176         _writer(F("  ===> %s") % title);
 177         for (std::vector< result_data >::const_iterator iter = all.begin();
 178              iter != all.end(); iter++) {
 179             _writer(F("%sok %s:%s%s") %
 180                 result %
 181                 (*iter).binary_path %
 182                 (*iter).test_case_name %
 183                 comment
 184                 );
 185             _writer(F("  %s:%s  ->  %s  [%s]") % (*iter).binary_path %
 186                     (*iter).test_case_name %
 187                     cli::format_result((*iter).result) %
 188                     cli::format_delta((*iter).duration));
 189         }
 190     }
 191
 192 public:
 193     /// Constructor for the hooks.
 194     ///
 195     /// \param ui_ The user interface object of the caller command.
 196     /// \param outfile_ The file to which to send the output.
 197     /// \param show_context_ Whether to include the runtime context in
 198     ///     the output or not.
 199     /// \param results_filters_ The result types to include in the report.
 200     ///     Cannot be empty.
 201     console_tap_hooks(cmdline::ui* ui_, const fs::path& outfile_,
 202                   const bool show_context_,
 203                   const cli::result_types& results_filters_) :
 204         _writer(ui_, outfile_),
 205         _show_context(show_context_),
 206         _results_filters(results_filters_)
 207     {
 208         PRE(!results_filters_.empty());
 209     }
 210
 211     /// Callback executed when an action is found.
 212     ///
 213     /// \param action_id The identifier of the loaded action.
 214     /// \param action The action loaded from the database.
 215     void
 216     got_action(const int64_t action_id, const engine::action& action)
 217     {
 218         _action_id = action_id;
 219         if (_show_context)
 220             print_context(action.runtime_context());
 221     }
 222
 223     /// Callback executed when a test results is found.
 224     ///
 225     /// \param iter Container for the test result's data.
 226     void
 227     got_result(store::results_iterator& iter)
 228     {
 229         _runtime += iter.duration();
 230         const engine::test_result result = iter.result();
 231         _results[result.type()].push_back(
 232             result_data(iter.test_program()->relative_path(),
 233                         iter.test_case_name(), iter.result(), iter.duration()));
 234     }
 235
 236     /// Prints the tests summary.
 237     void
 238     print_tests(void)
 239     {
 240         using engine::test_result;
 241         typedef std::map< test_result::result_type, const char* > types_map;
 242         typedef std::map< test_result::result_type, std::size_t > types_counts;
 243
 244         types_map titles;
 245         titles[engine::test_result::broken] = "Broken tests";
 246         titles[engine::test_result::expected_failure] = "Expected failures";
 247         titles[engine::test_result::failed] = "Failed tests";
 248         titles[engine::test_result::passed] = "Passed tests";
 249         titles[engine::test_result::skipped] = "Skipped tests";
 250
 251         types_counts counts;
 252         counts[engine::test_result::broken] = count_results(test_result::broken);
 253         counts[engine::test_result::expected_failure] = count_results(test_result::expected_failure);
 254         counts[engine::test_result::failed] = count_results(test_result::failed);
 255         counts[engine::test_result::passed] = count_results(test_result::passed);
 256         counts[engine::test_result::skipped] = count_results(test_result::skipped);
 257
 258         const std::size_t total =
 259             counts[engine::test_result::broken] +
 260             counts[engine::test_result::expected_failure] +
 261             counts[engine::test_result::failed] +
 262             counts[engine::test_result::passed] +
 263             counts[engine::test_result::skipped];
 264
 265         std::size_t selected_types_total = 0;
 266
 267         for (cli::result_types::const_iterator iter = _results_filters.begin();
 268              iter != _results_filters.end(); ++iter) {
 269             selected_types_total += counts.find(*iter)->second;
 270         }
 271
 272         _writer(F("1..%s") % selected_types_total);
 273         for (cli::result_types::const_iterator iter = _results_filters.begin();
 274              iter != _results_filters.end(); ++iter) {
 275             const types_map::const_iterator match = titles.find(*iter);
 276             INV_MSG(match != titles.end(), "Conditional does not match user "
 277                     "input validation in parse_types()");
 278             print_results((*match).first, (*match).second);
 279         }
 280
 281         _writer("  ===> Summary");
 282         _writer(F("  Action: %s") % _action_id);
 283         _writer(F("  Test cases: %s total, %s passed, %s skipped, %s expected failures, "
 284                   "%s broken, %s failed") % total %
 285                 counts[engine::test_result::passed] %
 286                 counts[engine::test_result::skipped] %
 287                 counts[engine::test_result::expected_failure] %
 288                 counts[engine::test_result::broken] %
 289                 counts[engine::test_result::failed]);
 290         _writer(F("  Total time: %s") % cli::format_delta(_runtime));
 291     }
 292 };
 293
 294
 295 }  // anonymous namespace
 296
 297
 298 /// Default constructor for cmd_report_tap.
 299 cmd_report_tap::cmd_report_tap(void) : cli_command(
 300     "report-tap", "", 0, 0,
 301     "Generates a TAP report with the result of a "
 302     "previous action")
 303 {
 304     add_option(store_option);
 305     add_option(cmdline::bool_option(
 306         "show-context", "Include the execution context in the report"));
 307     add_option(cmdline::int_option(
 308         "action", "The action to report; if not specified, defaults to the "
 309         "latest action in the database", "id"));
 310     add_option(cmdline::path_option(
 311         "output", "The file to which to write the report",
 312         "path", "/dev/stdout"));
 313     add_option(results_filter_option);
 314 }
 315
 316
 317 /// Entry point for the "report" subcommand.
 318 ///
 319 /// \param ui Object to interact with the I/O of the program.
 320 /// \param cmdline Representation of the command line to the subcommand.
 321 /// \param unused_user_config The runtime configuration of the program.
 322 ///
 323 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
 324 /// any other problem.
 325 int
 326 cmd_report_tap::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
 327                 const config::tree& UTILS_UNUSED_PARAM(user_config))
 328 {
 329     optional< int64_t > action_id;
 330     if (cmdline.has_option("action"))
 331         action_id = cmdline.get_option< cmdline::int_option >("action");
 332
 333     const result_types types = get_result_types(cmdline);
 334     console_tap_hooks hooks(
 335         ui, cmdline.get_option< cmdline::path_option >("output"),
 336         cmdline.has_option("show-context"), types);
 337     scan_action::drive(store_path(cmdline), action_id, hooks);
 338     hooks.print_tests();
 339
 340     return EXIT_SUCCESS;
 341 }