external/bsd/kyua-cli/dist/cli/cmd_report.cpp

   1 // Copyright 2011 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "cli/cmd_report.hpp"
  30
  31 #include <cstddef>
  32 #include <cstdlib>
  33 #include <fstream>
  34 #include <map>
  35 #include <vector>
  36
  37 #include "cli/common.ipp"
  38 #include "engine/action.hpp"
  39 #include "engine/context.hpp"
  40 #include "engine/drivers/scan_action.hpp"
  41 #include "engine/test_result.hpp"
  42 #include "utils/cmdline/exceptions.hpp"
  43 #include "utils/cmdline/parser.ipp"
  44 #include "utils/defs.hpp"
  45 #include "utils/format/macros.hpp"
  46 #include "utils/optional.ipp"
  47
  48 namespace cmdline = utils::cmdline;
  49 namespace config = utils::config;
  50 namespace datetime = utils::datetime;
  51 namespace fs = utils::fs;
  52 namespace scan_action = engine::drivers::scan_action;
  53
  54 using cli::cmd_report;
  55 using utils::optional;
  56
  57
  58 namespace {
  59
  60
  61 /// Generates a plain-text report intended to be printed to the console.
  62 class console_hooks : public scan_action::base_hooks {
  63     /// Indirection to print the output to the correct file stream.
  64     cli::file_writer _writer;
  65
  66     /// Whether to include the runtime context in the output or not.
  67     const bool _show_context;
  68
  69     /// Collection of result types to include in the report.
  70     const cli::result_types& _results_filters;
  71
  72     /// The action ID loaded.
  73     int64_t _action_id;
  74
  75     /// The total run time of the tests.
  76     datetime::delta _runtime;
  77
  78     /// Representation of a single result.
  79     struct result_data {
  80         /// The relative path to the test program.
  81         fs::path binary_path;
  82
  83         /// The name of the test case.
  84         std::string test_case_name;
  85
  86         /// The result of the test case.
  87         engine::test_result result;
  88
  89         /// The duration of the test case execution.
  90         datetime::delta duration;
  91
  92         /// Constructs a new results data.
  93         ///
  94         /// \param binary_path_ The relative path to the test program.
  95         /// \param test_case_name_ The name of the test case.
  96         /// \param result_ The result of the test case.
  97         /// \param duration_ The duration of the test case execution.
  98         result_data(const fs::path& binary_path_,
  99                     const std::string& test_case_name_,
 100                     const engine::test_result& result_,
 101                     const datetime::delta& duration_) :
 102             binary_path(binary_path_), test_case_name(test_case_name_),
 103             result(result_), duration(duration_)
 104         {
 105         }
 106     };
 107
 108     /// Results received, broken down by their type.
 109     ///
 110     /// Note that this may not include all results, as keeping the whole list in
 111     /// memory may be too much.
 112     std::map< engine::test_result::result_type,
 113               std::vector< result_data > > _results;
 114
 115     /// Prints the execution context to the output.
 116     ///
 117     /// \param context The context to dump.
 118     void
 119     print_context(const engine::context& context)
 120     {
 121         _writer("===> Execution context");
 122
 123         _writer(F("Current directory: %s") % context.cwd());
 124         const std::map< std::string, std::string >& env = context.env();
 125         if (env.empty())
 126             _writer("No environment variables recorded");
 127         else {
 128             _writer("Environment variables:");
 129             for (std::map< std::string, std::string >::const_iterator
 130                      iter = env.begin(); iter != env.end(); iter++) {
 131                 _writer(F("    %s=%s") % (*iter).first % (*iter).second);
 132             }
 133         }
 134     }
 135
 136     /// Counts how many results of a given type have been received.
 137     std::size_t
 138     count_results(const engine::test_result::result_type type)
 139     {
 140         const std::map< engine::test_result::result_type,
 141                         std::vector< result_data > >::const_iterator iter =
 142             _results.find(type);
 143         if (iter == _results.end())
 144             return 0;
 145         else
 146             return (*iter).second.size();
 147     }
 148
 149     /// Prints a set of results.
 150     void
 151     print_results(const engine::test_result::result_type type,
 152                   const char* title)
 153     {
 154         const std::map< engine::test_result::result_type,
 155                         std::vector< result_data > >::const_iterator iter2 =
 156             _results.find(type);
 157         if (iter2 == _results.end())
 158             return;
 159         const std::vector< result_data >& all = (*iter2).second;
 160
 161         _writer(F("===> %s") % title);
 162         for (std::vector< result_data >::const_iterator iter = all.begin();
 163              iter != all.end(); iter++) {
 164             _writer(F("%s:%s  ->  %s  [%s]") % (*iter).binary_path %
 165                     (*iter).test_case_name %
 166                     cli::format_result((*iter).result) %
 167                     cli::format_delta((*iter).duration));
 168         }
 169     }
 170
 171 public:
 172     /// Constructor for the hooks.
 173     ///
 174     /// \param ui_ The user interface object of the caller command.
 175     /// \param outfile_ The file to which to send the output.
 176     /// \param show_context_ Whether to include the runtime context in
 177     ///     the output or not.
 178     /// \param results_filters_ The result types to include in the report.
 179     ///     Cannot be empty.
 180     console_hooks(cmdline::ui* ui_, const fs::path& outfile_,
 181                   const bool show_context_,
 182                   const cli::result_types& results_filters_) :
 183         _writer(ui_, outfile_),
 184         _show_context(show_context_),
 185         _results_filters(results_filters_)
 186     {
 187         PRE(!results_filters_.empty());
 188     }
 189
 190     /// Callback executed when an action is found.
 191     ///
 192     /// \param action_id The identifier of the loaded action.
 193     /// \param action The action loaded from the database.
 194     void
 195     got_action(const int64_t action_id, const engine::action& action)
 196     {
 197         _action_id = action_id;
 198         if (_show_context)
 199             print_context(action.runtime_context());
 200     }
 201
 202     /// Callback executed when a test results is found.
 203     ///
 204     /// \param iter Container for the test result's data.
 205     void
 206     got_result(store::results_iterator& iter)
 207     {
 208         _runtime += iter.duration();
 209         const engine::test_result result = iter.result();
 210         _results[result.type()].push_back(
 211             result_data(iter.test_program()->relative_path(),
 212                         iter.test_case_name(), iter.result(), iter.duration()));
 213     }
 214
 215     /// Prints the tests summary.
 216     void
 217     print_tests(void)
 218     {
 219         using engine::test_result;
 220         typedef std::map< test_result::result_type, const char* > types_map;
 221
 222         types_map titles;
 223         titles[engine::test_result::broken] = "Broken tests";
 224         titles[engine::test_result::expected_failure] = "Expected failures";
 225         titles[engine::test_result::failed] = "Failed tests";
 226         titles[engine::test_result::passed] = "Passed tests";
 227         titles[engine::test_result::skipped] = "Skipped tests";
 228
 229         for (cli::result_types::const_iterator iter = _results_filters.begin();
 230              iter != _results_filters.end(); ++iter) {
 231             const types_map::const_iterator match = titles.find(*iter);
 232             INV_MSG(match != titles.end(), "Conditional does not match user "
 233                     "input validation in parse_types()");
 234             print_results((*match).first, (*match).second);
 235         }
 236
 237         const std::size_t broken = count_results(test_result::broken);
 238         const std::size_t failed = count_results(test_result::failed);
 239         const std::size_t passed = count_results(test_result::passed);
 240         const std::size_t skipped = count_results(test_result::skipped);
 241         const std::size_t xfail = count_results(test_result::expected_failure);
 242         const std::size_t total = broken + failed + passed + skipped + xfail;
 243
 244         _writer("===> Summary");
 245         _writer(F("Action: %s") % _action_id);
 246         _writer(F("Test cases: %s total, %s skipped, %s expected failures, "
 247                   "%s broken, %s failed") %
 248                 total % skipped % xfail % broken % failed);
 249         _writer(F("Total time: %s") % cli::format_delta(_runtime));
 250     }
 251 };
 252
 253
 254 }  // anonymous namespace
 255
 256
 257 const fs::path cli::file_writer::_stdout_path("/dev/stdout");
 258 const fs::path cli::file_writer::_stderr_path("/dev/stderr");
 259
 260
 261 /// Constructs a new file_writer wrapper.
 262 ///
 263 /// \param ui_ The UI object of the caller command.
 264 /// \param path_ The path to the output file.
 265 cli::file_writer::file_writer(cmdline::ui* const ui_, const fs::path& path_) :
 266     _ui(ui_), _output_path(path_)
 267 {
 268     if (path_ != _stdout_path && path_ != _stderr_path) {
 269         _output_file.reset(new std::ofstream(path_.c_str()));
 270         if (!*(_output_file)) {
 271             throw std::runtime_error(F("Cannot open output file %s") % path_);
 272         }
 273     }
 274 }
 275
 276 /// Destructor.
 277 cli::file_writer::~file_writer(void)
 278 {
 279 }
 280
 281 /// Writes a message to the selected output.
 282 ///
 283 /// \param message The message to write; should not include a termination
 284 ///     new line.
 285 void
 286 cli::file_writer::operator()(const std::string& message)
 287 {
 288     if (_output_path == _stdout_path)
 289         _ui->out(message);
 290     else if (_output_path == _stderr_path)
 291         _ui->err(message);
 292     else {
 293         INV(_output_file.get() != NULL);
 294         (*_output_file) << message << '\n';
 295     }
 296 }
 297
 298
 299 /// Default constructor for cmd_report.
 300 cmd_report::cmd_report(void) : cli_command(
 301     "report", "", 0, 0,
 302     "Generates a user-friendly, plain-text report with the result of a "
 303     "previous action")
 304 {
 305     add_option(store_option);
 306     add_option(cmdline::bool_option(
 307         "show-context", "Include the execution context in the report"));
 308     add_option(cmdline::int_option(
 309         "action", "The action to report; if not specified, defaults to the "
 310         "latest action in the database", "id"));
 311     add_option(cmdline::path_option(
 312         "output", "The file to which to write the report",
 313         "path", "/dev/stdout"));
 314     add_option(results_filter_option);
 315 }
 316
 317
 318 /// Entry point for the "report" subcommand.
 319 ///
 320 /// \param ui Object to interact with the I/O of the program.
 321 /// \param cmdline Representation of the command line to the subcommand.
 322 /// \param unused_user_config The runtime configuration of the program.
 323 ///
 324 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
 325 /// any other problem.
 326 int
 327 cmd_report::run(cmdline::ui* ui, const cmdline::parsed_cmdline& cmdline,
 328                 const config::tree& UTILS_UNUSED_PARAM(user_config))
 329 {
 330     optional< int64_t > action_id;
 331     if (cmdline.has_option("action"))
 332         action_id = cmdline.get_option< cmdline::int_option >("action");
 333
 334     const result_types types = get_result_types(cmdline);
 335     console_hooks hooks(
 336         ui, cmdline.get_option< cmdline::path_option >("output"),
 337         cmdline.has_option("show-context"), types);
 338     scan_action::drive(store_path(cmdline), action_id, hooks);
 339     hooks.print_tests();
 340
 341     return EXIT_SUCCESS;
 342 }