external/bsd/kyua-cli/dist/cli/cmd_report_html.cpp

   1 // Copyright 2012 Google Inc.
   2 // All rights reserved.
   3 //
   4 // Redistribution and use in source and binary forms, with or without
   5 // modification, are permitted provided that the following conditions are
   6 // met:
   7 //
   8 // * Redistributions of source code must retain the above copyright
   9 //   notice, this list of conditions and the following disclaimer.
  10 // * Redistributions in binary form must reproduce the above copyright
  11 //   notice, this list of conditions and the following disclaimer in the
  12 //   documentation and/or other materials provided with the distribution.
  13 // * Neither the name of Google Inc. nor the names of its contributors
  14 //   may be used to endorse or promote products derived from this software
  15 //   without specific prior written permission.
  16 //
  17 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 #include "cli/cmd_report_html.hpp"
  30
  31 #include <algorithm>
  32 #include <cerrno>
  33 #include <cstdlib>
  34 #include <stdexcept>
  35
  36 #include "cli/common.ipp"
  37 #include "engine/action.hpp"
  38 #include "engine/context.hpp"
  39 #include "engine/drivers/scan_action.hpp"
  40 #include "engine/test_result.hpp"
  41 #include "utils/cmdline/options.hpp"
  42 #include "utils/cmdline/parser.ipp"
  43 #include "utils/datetime.hpp"
  44 #include "utils/env.hpp"
  45 #include "utils/format/macros.hpp"
  46 #include "utils/fs/exceptions.hpp"
  47 #include "utils/fs/operations.hpp"
  48 #include "utils/fs/path.hpp"
  49 #include "utils/optional.ipp"
  50 #include "utils/text/templates.hpp"
  51
  52 namespace cmdline = utils::cmdline;
  53 namespace config = utils::config;
  54 namespace datetime = utils::datetime;
  55 namespace fs = utils::fs;
  56 namespace scan_action = engine::drivers::scan_action;
  57 namespace text = utils::text;
  58
  59 using utils::optional;
  60
  61
  62 namespace {
  63
  64
  65 /// Creates the report's top directory and fails if it exists.
  66 ///
  67 /// \param directory The directory to create.
  68 /// \param force Whether to wipe an existing directory or not.
  69 ///
  70 /// \throw std::runtime_error If the directory already exists; this is a user
  71 ///     error that the user must correct.
  72 /// \throw fs::error If the directory creation fails for any other reason.
  73 static void
  74 create_top_directory(const fs::path& directory, const bool force)
  75 {
  76     if (force) {
  77         if (fs::exists(directory))
  78             fs::rm_r(directory);
  79     }
  80
  81     try {
  82         fs::mkdir(directory, 0755);
  83     } catch (const fs::system_error& e) {
  84         if (e.original_errno() == EEXIST)
  85             throw std::runtime_error(F("Output directory '%s' already exists; "
  86                                        "maybe use --force?") %
  87                                      directory);
  88         else
  89             throw e;
  90     }
  91 }
  92
  93
  94 /// Generates a flat unique filename for a given test case.
  95 ///
  96 /// \param test_case The test case for which to genereate the name.
  97 ///
  98 /// \return A filename unique within a directory with a trailing HTML extension.
  99 static std::string
 100 test_case_filename(const engine::test_case& test_case)
 101 {
 102     static const char* special_characters = "/:";
 103
 104     std::string name = cli::format_test_case_id(test_case);
 105     std::string::size_type pos = name.find_first_of(special_characters);
 106     while (pos != std::string::npos) {
 107         name.replace(pos, 1, "_");
 108         pos = name.find_first_of(special_characters, pos + 1);
 109     }
 110     return name + ".html";
 111 }
 112
 113
 114 /// Adds a string to string map to the templates.
 115 ///
 116 /// \param [in,out] templates The templates to add the map to.
 117 /// \param props The map to add to the templates.
 118 /// \param key_vector Name of the template vector that holds the keys.
 119 /// \param value_vector Name of the template vector that holds the values.
 120 static void
 121 add_map(text::templates_def& templates, const config::properties_map& props,
 122         const std::string& key_vector, const std::string& value_vector)
 123 {
 124     templates.add_vector(key_vector);
 125     templates.add_vector(value_vector);
 126
 127     for (config::properties_map::const_iterator iter = props.begin();
 128          iter != props.end(); ++iter) {
 129         templates.add_to_vector(key_vector, (*iter).first);
 130         templates.add_to_vector(value_vector, (*iter).second);
 131     }
 132 }
 133
 134
 135 /// Generates an HTML report.
 136 class html_hooks : public scan_action::base_hooks {
 137     /// User interface object where to report progress.
 138     cmdline::ui* _ui;
 139
 140     /// The top directory in which to create the HTML files.
 141     fs::path _directory;
 142
 143     /// Collection of result types to include in the report.
 144     const cli::result_types& _results_filters;
 145
 146     /// Templates accumulator to generate the index.html file.
 147     text::templates_def _summary_templates;
 148
 149     /// Mapping of result types to the amount of tests with such result.
 150     std::map< engine::test_result::result_type, std::size_t > _types_count;
 151
 152     /// Generates a common set of templates for all of our files.
 153     ///
 154     /// \return A new templates object with common parameters.
 155     static text::templates_def
 156     common_templates(void)
 157     {
 158         text::templates_def templates;
 159         templates.add_variable("css", "report.css");
 160         return templates;
 161     }
 162
 163     /// Adds a test case result to the summary.
 164     ///
 165     /// \param test_case The test case to be added.
 166     /// \param result The result of the test case.
 167     /// \param has_detail If true, the result of the test case has not been
 168     ///     filtered and therefore there exists a separate file for the test
 169     ///     with all of its information.
 170     void
 171     add_to_summary(const engine::test_case& test_case,
 172                    const engine::test_result& result,
 173                    const bool has_detail)
 174     {
 175         ++_types_count[result.type()];
 176
 177         if (!has_detail)
 178             return;
 179
 180         std::string test_cases_vector;
 181         std::string test_cases_file_vector;
 182         switch (result.type()) {
 183         case engine::test_result::broken:
 184             test_cases_vector = "broken_test_cases";
 185             test_cases_file_vector = "broken_test_cases_file";
 186             break;
 187
 188         case engine::test_result::expected_failure:
 189             test_cases_vector = "xfail_test_cases";
 190             test_cases_file_vector = "xfail_test_cases_file";
 191             break;
 192
 193         case engine::test_result::failed:
 194             test_cases_vector = "failed_test_cases";
 195             test_cases_file_vector = "failed_test_cases_file";
 196             break;
 197
 198         case engine::test_result::passed:
 199             test_cases_vector = "passed_test_cases";
 200             test_cases_file_vector = "passed_test_cases_file";
 201             break;
 202
 203         case engine::test_result::skipped:
 204             test_cases_vector = "skipped_test_cases";
 205             test_cases_file_vector = "skipped_test_cases_file";
 206             break;
 207         }
 208         INV(!test_cases_vector.empty());
 209         INV(!test_cases_file_vector.empty());
 210
 211         _summary_templates.add_to_vector(test_cases_vector,
 212                                          cli::format_test_case_id(test_case));
 213         _summary_templates.add_to_vector(test_cases_file_vector,
 214                                          test_case_filename(test_case));
 215     }
 216
 217     /// Instantiate a template to generate an HTML file in the output directory.
 218     ///
 219     /// \param templates The templates to use.
 220     /// \param template_name The name of the template.  This is automatically
 221     ///     searched for in the installed directory, so do not provide a path.
 222     /// \param output_name The name of the output file.  This is a basename to
 223     ///     be created within the output directory.
 224     ///
 225     /// \throw text::error If there is any problem applying the templates.
 226     void
 227     generate(const text::templates_def& templates,
 228              const std::string& template_name,
 229              const std::string& output_name) const
 230     {
 231         const fs::path miscdir(utils::getenv_with_default(
 232              "KYUA_MISCDIR", KYUA_MISCDIR));
 233         const fs::path template_file = miscdir / template_name;
 234         const fs::path output_path(_directory / output_name);
 235
 236         _ui->out(F("Generating %s") % output_path);
 237         text::instantiate(templates, template_file, output_path);
 238     }
 239
 240     /// Gets the number of tests with a given result type.
 241     ///
 242     /// \param type The type to be queried.
 243     ///
 244     /// \return The number of tests of the given type, or 0 if none have yet
 245     /// been registered by add_to_summary().
 246     std::size_t
 247     get_count(const engine::test_result::result_type type) const
 248     {
 249         const std::map< engine::test_result::result_type,
 250                         std::size_t >::const_iterator
 251             iter = _types_count.find(type);
 252         if (iter == _types_count.end())
 253             return 0;
 254         else
 255             return (*iter).second;
 256     }
 257
 258 public:
 259     /// Constructor for the hooks.
 260     ///
 261     /// \param ui_ User interface object where to report progress.
 262     /// \param directory_ The directory in which to create the HTML files.
 263     /// \param results_filters_ The result types to include in the report.
 264     ///     Cannot be empty.
 265     html_hooks(cmdline::ui* ui_, const fs::path& directory_,
 266                const cli::result_types& results_filters_) :
 267         _ui(ui_),
 268         _directory(directory_),
 269         _results_filters(results_filters_),
 270         _summary_templates(common_templates())
 271     {
 272         PRE(!results_filters_.empty());
 273
 274         // Keep in sync with add_to_summary().
 275         _summary_templates.add_vector("broken_test_cases");
 276         _summary_templates.add_vector("broken_test_cases_file");
 277         _summary_templates.add_vector("xfail_test_cases");
 278         _summary_templates.add_vector("xfail_test_cases_file");
 279         _summary_templates.add_vector("failed_test_cases");
 280         _summary_templates.add_vector("failed_test_cases_file");
 281         _summary_templates.add_vector("passed_test_cases");
 282         _summary_templates.add_vector("passed_test_cases_file");
 283         _summary_templates.add_vector("skipped_test_cases");
 284         _summary_templates.add_vector("skipped_test_cases_file");
 285     }
 286
 287     /// Callback executed when an action is found.
 288     ///
 289     /// \param action_id The identifier of the loaded action.
 290     /// \param action The action loaded from the database.
 291     void
 292     got_action(const int64_t action_id,
 293                const engine::action& action)
 294     {
 295         _summary_templates.add_variable("action_id", F("%s") % action_id);
 296
 297         const engine::context& context = action.runtime_context();
 298         text::templates_def templates = common_templates();
 299         templates.add_variable("action_id", F("%s") % action_id);
 300         templates.add_variable("cwd", context.cwd().str());
 301         add_map(templates, context.env(), "env_var", "env_var_value");
 302         generate(templates, "context.html", "context.html");
 303     }
 304
 305     /// Callback executed when a test results is found.
 306     ///
 307     /// \param iter Container for the test result's data.
 308     void
 309     got_result(store::results_iterator& iter)
 310     {
 311         const engine::test_program_ptr test_program = iter.test_program();
 312         const engine::test_result result = iter.result();
 313
 314         const engine::test_case& test_case = *test_program->find(
 315             iter.test_case_name());
 316
 317         if (std::find(_results_filters.begin(), _results_filters.end(),
 318                       result.type()) == _results_filters.end()) {
 319             add_to_summary(test_case, result, false);
 320             return;
 321         }
 322
 323         add_to_summary(test_case, result, true);
 324
 325         text::templates_def templates = common_templates();
 326         templates.add_variable("test_case",
 327                                cli::format_test_case_id(test_case));
 328         templates.add_variable("test_program",
 329                                test_program->absolute_path().str());
 330         templates.add_variable("result", cli::format_result(result));
 331         templates.add_variable("duration", cli::format_delta(iter.duration()));
 332
 333         add_map(templates, test_case.get_metadata().to_properties(),
 334                 "metadata_var", "metadata_value");
 335
 336         {
 337             const std::string stdout_text = iter.stdout_contents();
 338             if (!stdout_text.empty())
 339                 templates.add_variable("stdout", stdout_text);
 340         }
 341         {
 342             const std::string stderr_text = iter.stderr_contents();
 343             if (!stderr_text.empty())
 344                 templates.add_variable("stderr", stderr_text);
 345         }
 346
 347         generate(templates, "test_result.html", test_case_filename(test_case));
 348     }
 349
 350     /// Writes the index.html file in the output directory.
 351     ///
 352     /// This should only be called once all the processing has been done;
 353     /// i.e. when the scan_action driver returns.
 354     void
 355     write_summary(void)
 356     {
 357         const std::size_t n_passed = get_count(engine::test_result::passed);
 358         const std::size_t n_failed = get_count(engine::test_result::failed);
 359         const std::size_t n_skipped = get_count(engine::test_result::skipped);
 360         const std::size_t n_xfail = get_count(
 361             engine::test_result::expected_failure);
 362         const std::size_t n_broken = get_count(engine::test_result::broken);
 363
 364         const std::size_t n_bad = n_broken + n_failed;
 365
 366         _summary_templates.add_variable("passed_tests_count",
 367                                         F("%s") % n_passed);
 368         _summary_templates.add_variable("failed_tests_count",
 369                                         F("%s") % n_failed);
 370         _summary_templates.add_variable("skipped_tests_count",
 371                                         F("%s") % n_skipped);
 372         _summary_templates.add_variable("xfail_tests_count",
 373                                         F("%s") % n_xfail);
 374         _summary_templates.add_variable("broken_tests_count",
 375                                         F("%s") % n_broken);
 376         _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad);
 377
 378         generate(text::templates_def(), "report.css", "report.css");
 379         generate(_summary_templates, "index.html", "index.html");
 380     }
 381 };
 382
 383
 384 }  // anonymous namespace
 385
 386
 387 /// Default constructor for cmd_report_html.
 388 cli::cmd_report_html::cmd_report_html(void) : cli_command(
 389     "report-html", "", 0, 0,
 390     "Generates an HTML report with the result of a previous action")
 391 {
 392     add_option(store_option);
 393     add_option(cmdline::int_option(
 394         "action", "The action to report; if not specified, defaults to the "
 395         "latest action in the database", "id"));
 396     add_option(cmdline::bool_option(
 397         "force", "Wipe the output directory before generating the new report; "
 398         "use care"));
 399     add_option(cmdline::path_option(
 400         "output", "The directory in which to store the HTML files",
 401         "path", "html"));
 402     add_option(cmdline::list_option(
 403         "results-filter", "Comma-separated list of result types to include in "
 404         "the report", "types", "skipped,xfail,broken,failed"));
 405 }
 406
 407
 408 /// Entry point for the "report-html" subcommand.
 409 ///
 410 /// \param ui Object to interact with the I/O of the program.
 411 /// \param cmdline Representation of the command line to the subcommand.
 412 /// \param unused_user_config The runtime configuration of the program.
 413 ///
 414 /// \return 0 if everything is OK, 1 if the statement is invalid or if there is
 415 /// any other problem.
 416 int
 417 cli::cmd_report_html::run(cmdline::ui* ui,
 418                           const cmdline::parsed_cmdline& cmdline,
 419                           const config::tree& UTILS_UNUSED_PARAM(user_config))
 420 {
 421     optional< int64_t > action_id;
 422     if (cmdline.has_option("action"))
 423         action_id = cmdline.get_option< cmdline::int_option >("action");
 424
 425     const result_types types = get_result_types(cmdline);
 426     const fs::path directory =
 427         cmdline.get_option< cmdline::path_option >("output");
 428     create_top_directory(directory, cmdline.has_option("force"));
 429     html_hooks hooks(ui, directory, types);
 430     scan_action::drive(store_path(cmdline), action_id, hooks);
 431     hooks.write_summary();
 432
 433     return EXIT_SUCCESS;
 434 }