ppapi/tests/testing_instance.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "ppapi/tests/testing_instance.h"
   6
   7 #include <algorithm>
   8 #include <cstring>
   9 #include <iomanip>
  10 #include <sstream>
  11 #include <vector>
  12
  13 #include "ppapi/cpp/core.h"
  14 #include "ppapi/cpp/module.h"
  15 #include "ppapi/cpp/var.h"
  16 #include "ppapi/cpp/view.h"
  17 #include "ppapi/tests/test_case.h"
  18
  19 TestCaseFactory* TestCaseFactory::head_ = NULL;
  20
  21 // Cookie value we use to signal "we're still working." See the comment above
  22 // the class declaration for how this works.
  23 static const char kProgressSignal[] = "...";
  24
  25 // Returns a new heap-allocated test case for the given test, or NULL on
  26 // failure.
  27 TestingInstance::TestingInstance(PP_Instance instance)
  28 #if (defined __native_client__)
  29     : pp::Instance(instance),
  30 #else
  31     : pp::InstancePrivate(instance),
  32 #endif
  33       current_case_(NULL),
  34       executed_tests_(false),
  35       number_tests_executed_(0),
  36       nacl_mode_(false),
  37       ssl_server_port_(-1),
  38       websocket_port_(-1),
  39       remove_plugin_(true) {
  40   callback_factory_.Initialize(this);
  41 }
  42
  43 TestingInstance::~TestingInstance() {
  44   if (current_case_)
  45     delete current_case_;
  46 }
  47
  48 bool TestingInstance::Init(uint32_t argc,
  49                            const char* argn[],
  50                            const char* argv[]) {
  51   for (uint32_t i = 0; i < argc; i++) {
  52     if (std::strcmp(argn[i], "mode") == 0) {
  53       if (std::strcmp(argv[i], "nacl") == 0)
  54         nacl_mode_ = true;
  55     } else if (std::strcmp(argn[i], "protocol") == 0) {
  56       protocol_ = argv[i];
  57     } else if (std::strcmp(argn[i], "websocket_host") == 0) {
  58       websocket_host_ = argv[i];
  59     } else if (std::strcmp(argn[i], "websocket_port") == 0) {
  60       websocket_port_ = atoi(argv[i]);
  61     } else if (std::strcmp(argn[i], "ssl_server_port") == 0) {
  62       ssl_server_port_ = atoi(argv[i]);
  63     }
  64   }
  65   // Create the proper test case from the argument.
  66   for (uint32_t i = 0; i < argc; i++) {
  67     if (std::strcmp(argn[i], "testcase") == 0) {
  68       if (argv[i][0] == '\0')
  69         break;
  70       current_case_ = CaseForTestName(argv[i]);
  71       test_filter_ = argv[i];
  72       if (!current_case_)
  73         errors_.append(std::string("Unknown test case ") + argv[i]);
  74       else if (!current_case_->Init())
  75         errors_.append(" Test case could not initialize.");
  76       return true;
  77     }
  78   }
  79
  80   // In DidChangeView, we'll dump out a list of all available tests.
  81   return true;
  82 }
  83
  84 #if !(defined __native_client__)
  85 pp::Var TestingInstance::GetInstanceObject() {
  86   if (current_case_)
  87     return current_case_->GetTestObject();
  88
  89   return pp::VarPrivate();
  90 }
  91 #endif
  92
  93 void TestingInstance::HandleMessage(const pp::Var& message_data) {
  94   if (current_case_)
  95     current_case_->HandleMessage(message_data);
  96 }
  97
  98 void TestingInstance::DidChangeView(const pp::View& view) {
  99   if (!executed_tests_) {
 100     executed_tests_ = true;
 101     pp::Module::Get()->core()->CallOnMainThread(
 102         0,
 103         callback_factory_.NewCallback(&TestingInstance::ExecuteTests));
 104   }
 105   if (current_case_)
 106     current_case_->DidChangeView(view);
 107 }
 108
 109 bool TestingInstance::HandleInputEvent(const pp::InputEvent& event) {
 110   if (current_case_)
 111     return current_case_->HandleInputEvent(event);
 112   return false;
 113 }
 114
 115 void TestingInstance::EvalScript(const std::string& script) {
 116   SendTestCommand("EvalScript", script);
 117 }
 118
 119 void TestingInstance::SetCookie(const std::string& name,
 120                                 const std::string& value) {
 121   SendTestCommand("SetCookie", name + "=" + value);
 122 }
 123
 124 void TestingInstance::LogTest(const std::string& test_name,
 125                               const std::string& error_message,
 126                               PP_TimeTicks start_time) {
 127   current_test_name_ = test_name;
 128
 129   // Compute the time to run the test and save it in a string for logging:
 130   PP_TimeTicks end_time(pp::Module::Get()->core()->GetTimeTicks());
 131   std::ostringstream number_stream;
 132   PP_TimeTicks elapsed_time(end_time - start_time);
 133   number_stream << std::fixed << std::setprecision(3) << elapsed_time;
 134   std::string time_string(number_stream.str());
 135
 136   // Tell the browser we're still working.
 137   ReportProgress(kProgressSignal);
 138
 139   number_tests_executed_++;
 140
 141   std::string html;
 142   html.append("<div class=\"test_line\"><span class=\"test_name\">");
 143   html.append(test_name);
 144   html.append("</span> ");
 145   if (error_message.empty()) {
 146     html.append("<span class=\"pass\">PASS</span>");
 147   } else {
 148     html.append("<span class=\"fail\">FAIL</span>: <span class=\"err_msg\">");
 149     html.append(error_message);
 150     html.append("</span>");
 151
 152     if (!errors_.empty())
 153       errors_.append(", ");  // Separator for different error messages.
 154     errors_.append(test_name + " FAIL: " + error_message);
 155   }
 156   html.append(" <span class=\"time\">(");
 157   html.append(time_string);
 158   html.append("s)</span>");
 159
 160   html.append("</div>");
 161   LogHTML(html);
 162
 163   std::string test_time;
 164   test_time.append(test_name);
 165   test_time.append(" finished in ");
 166   test_time.append(time_string);
 167   test_time.append(" seconds.");
 168   LogTestTime(test_time);
 169
 170   current_test_name_.clear();
 171 }
 172
 173 void TestingInstance::AppendError(const std::string& message) {
 174   if (!errors_.empty())
 175     errors_.append(", ");
 176   errors_.append(message);
 177 }
 178
 179 void TestingInstance::ExecuteTests(int32_t unused) {
 180   ReportProgress(kProgressSignal);
 181
 182   // Clear the console.
 183   SendTestCommand("ClearConsole");
 184
 185   if (!errors_.empty()) {
 186     // Catch initialization errors and output the current error string to
 187     // the console.
 188     LogError("Plugin initialization failed: " + errors_);
 189   } else if (!current_case_) {
 190     LogAvailableTests();
 191     errors_.append("FAIL: Only listed tests");
 192   } else {
 193     current_case_->RunTests(test_filter_);
 194
 195     if (number_tests_executed_ == 0) {
 196       errors_.append("No tests executed. The test filter might be too "
 197                      "restrictive: '" + test_filter_ + "'.");
 198       LogError(errors_);
 199     }
 200     if (current_case_->skipped_tests().size()) {
 201       // TODO(dmichael): Convert all TestCases to run all tests in one fixture,
 202       //                 and enable this check. Currently, a lot of our tests
 203       //                 run 1 test per fixture, which is slow.
 204       /*
 205       errors_.append("Some tests were not listed and thus were not run. Make "
 206                      "sure all tests are passed in the test_case URL (even if "
 207                      "they are marked DISABLED_). Forgotten tests: ");
 208       std::set<std::string>::const_iterator iter =
 209           current_case_->skipped_tests().begin();
 210       for (; iter != current_case_->skipped_tests().end(); ++iter) {
 211         errors_.append(*iter);
 212         errors_.append(" ");
 213       }
 214       LogError(errors_);
 215       */
 216     }
 217     if (current_case_->remaining_tests().size()) {
 218       errors_.append("Some listed tests were not found in the TestCase. Check "
 219                      "the test names that were passed to make sure they match "
 220                      "tests in the TestCase. Unknown tests: ");
 221       std::map<std::string, bool>::const_iterator iter =
 222           current_case_->remaining_tests().begin();
 223       for (; iter != current_case_->remaining_tests().end(); ++iter) {
 224         errors_.append(iter->first);
 225         errors_.append(" ");
 226       }
 227       LogError(errors_);
 228     }
 229   }
 230
 231   if (remove_plugin_)
 232     SendTestCommand("RemovePluginWhenFinished");
 233   std::string result(errors_);
 234   if (result.empty())
 235     result = "PASS";
 236   SendTestCommand("DidExecuteTests", result);
 237   // Note, DidExecuteTests may unload the plugin. We can't really do anything
 238   // after this point.
 239 }
 240
 241 TestCase* TestingInstance::CaseForTestName(const std::string& name) {
 242   std::string case_name = name.substr(0, name.find_first_of('_'));
 243   TestCaseFactory* iter = TestCaseFactory::head_;
 244   while (iter != NULL) {
 245     if (case_name == iter->name_)
 246       return iter->method_(this);
 247     iter = iter->next_;
 248   }
 249   return NULL;
 250 }
 251
 252 void TestingInstance::SendTestCommand(const std::string& command) {
 253   std::string msg("TESTING_MESSAGE:");
 254   msg += command;
 255   PostMessage(pp::Var(msg));
 256 }
 257
 258 void TestingInstance::SendTestCommand(const std::string& command,
 259                                       const std::string& params) {
 260   SendTestCommand(command + ":" + params);
 261 }
 262
 263
 264 void TestingInstance::LogAvailableTests() {
 265   // Print out a listing of all tests.
 266   std::vector<std::string> test_cases;
 267   TestCaseFactory* iter = TestCaseFactory::head_;
 268   while (iter != NULL) {
 269     test_cases.push_back(iter->name_);
 270     iter = iter->next_;
 271   }
 272   std::sort(test_cases.begin(), test_cases.end());
 273
 274   std::string html;
 275   html.append("Available test cases: <dl>");
 276   for (size_t i = 0; i < test_cases.size(); ++i) {
 277     html.append("<dd><a href='?testcase=");
 278     html.append(test_cases[i]);
 279     if (nacl_mode_)
 280        html.append("&mode=nacl");
 281     html.append("'>");
 282     html.append(test_cases[i]);
 283     html.append("</a></dd>");
 284   }
 285   html.append("</dl>");
 286   html.append("<button onclick='RunAll()'>Run All Tests</button>");
 287
 288   LogHTML(html);
 289 }
 290
 291 void TestingInstance::LogError(const std::string& text) {
 292   std::string html;
 293   html.append("<span class=\"fail\">FAIL</span>: <span class=\"err_msg\">");
 294   html.append(text);
 295   html.append("</span>");
 296   LogHTML(html);
 297 }
 298
 299 void TestingInstance::LogHTML(const std::string& html) {
 300   SendTestCommand("LogHTML", html);
 301 }
 302
 303 void TestingInstance::ReportProgress(const std::string& progress_value) {
 304   SendTestCommand("ReportProgress", progress_value);
 305 }
 306
 307 void TestingInstance::AddPostCondition(const std::string& script) {
 308   SendTestCommand("AddPostCondition", script);
 309 }
 310
 311 void TestingInstance::LogTestTime(const std::string& test_time) {
 312   SendTestCommand("LogTestTime", test_time);
 313 }
 314
 315 class Module : public pp::Module {
 316  public:
 317   Module() : pp::Module() {}
 318   virtual ~Module() {}
 319
 320   virtual pp::Instance* CreateInstance(PP_Instance instance) {
 321     return new TestingInstance(instance);
 322   }
 323 };
 324
 325 namespace pp {
 326
 327 Module* CreateModule() {
 328   return new ::Module();
 329 }
 330
 331 }  // namespace pp