Upstreaming browser/ui/uikit_ui_util from iOS.
[chromium-blink-merge.git] / content / public / test / test_launcher.cc
blob63532ee102e92cf2c6a55b2dbc6305412444c037
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/public/test/test_launcher.h"
7 #include <map>
8 #include <string>
9 #include <vector>
11 #include "base/command_line.h"
12 #include "base/containers/hash_tables.h"
13 #include "base/environment.h"
14 #include "base/files/file_util.h"
15 #include "base/files/scoped_temp_dir.h"
16 #include "base/logging.h"
17 #include "base/memory/linked_ptr.h"
18 #include "base/memory/scoped_ptr.h"
19 #include "base/message_loop/message_loop.h"
20 #include "base/stl_util.h"
21 #include "base/strings/string_number_conversions.h"
22 #include "base/strings/string_util.h"
23 #include "base/strings/utf_string_conversions.h"
24 #include "base/test/launcher/test_launcher.h"
25 #include "base/test/test_suite.h"
26 #include "base/test/test_switches.h"
27 #include "base/test/test_timeouts.h"
28 #include "base/time/time.h"
29 #include "content/public/app/content_main.h"
30 #include "content/public/app/content_main_delegate.h"
31 #include "content/public/app/startup_helper_win.h"
32 #include "content/public/common/content_switches.h"
33 #include "content/public/common/sandbox_init.h"
34 #include "content/public/test/browser_test.h"
35 #include "net/base/escape.h"
36 #include "testing/gtest/include/gtest/gtest.h"
38 #if defined(OS_WIN)
39 #include "base/base_switches.h"
40 #include "content/common/sandbox_win.h"
41 #include "sandbox/win/src/sandbox_factory.h"
42 #include "sandbox/win/src/sandbox_types.h"
43 #elif defined(OS_MACOSX)
44 #include "base/mac/scoped_nsautorelease_pool.h"
45 #endif
47 namespace content {
49 namespace {
51 // Tests with this prefix run before the same test without it, and use the same
52 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
53 // that span browser restarts.
54 const char kPreTestPrefix[] = "PRE_";
56 // Manual tests only run when --run-manual is specified. This allows writing
57 // tests that don't run automatically but are still in the same test binary.
58 // This is useful so that a team that wants to run a few tests doesn't have to
59 // add a new binary that must be compiled on all builds.
60 const char kManualTestPrefix[] = "MANUAL_";
62 TestLauncherDelegate* g_launcher_delegate;
63 ContentMainParams* g_params;
65 std::string RemoveAnyPrePrefixes(const std::string& test_name) {
66 std::string result(test_name);
67 base::ReplaceSubstringsAfterOffset(
68 &result, 0, kPreTestPrefix, base::StringPiece());
69 return result;
72 void PrintUsage() {
73 fprintf(stdout,
74 "Runs tests using the gtest framework, each batch of tests being\n"
75 "run in their own process. Supported command-line flags:\n"
76 "\n"
77 " Common flags:\n"
78 " --gtest_filter=...\n"
79 " Runs a subset of tests (see --gtest_help for more info).\n"
80 "\n"
81 " --help\n"
82 " Shows this message.\n"
83 "\n"
84 " --gtest_help\n"
85 " Shows the gtest help message.\n"
86 "\n"
87 " --test-launcher-jobs=N\n"
88 " Sets the number of parallel test jobs to N.\n"
89 "\n"
90 " --single_process\n"
91 " Runs the tests and the launcher in the same process. Useful\n"
92 " for debugging a specific test in a debugger.\n"
93 "\n"
94 " Other flags:\n"
95 " --test-launcher-retry-limit=N\n"
96 " Sets the limit of test retries on failures to N.\n"
97 "\n"
98 " --test-launcher-summary-output=PATH\n"
99 " Saves a JSON machine-readable summary of the run.\n"
100 "\n"
101 " --test-launcher-print-test-stdio=auto|always|never\n"
102 " Controls when full test output is printed.\n"
103 " auto means to print it when the test failed.\n"
104 "\n"
105 " --test-launcher-total-shards=N\n"
106 " Sets the total number of shards to N.\n"
107 "\n"
108 " --test-launcher-shard-index=N\n"
109 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
112 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
113 // wrapping a lower-level test launcher with content-specific code.
114 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
115 public:
116 explicit WrapperTestLauncherDelegate(
117 content::TestLauncherDelegate* launcher_delegate)
118 : launcher_delegate_(launcher_delegate) {
119 CHECK(temp_dir_.CreateUniqueTempDir());
122 // base::TestLauncherDelegate:
123 bool GetTests(std::vector<base::TestIdentifier>* output) override;
124 bool ShouldRunTest(const std::string& test_case_name,
125 const std::string& test_name) override;
126 size_t RunTests(base::TestLauncher* test_launcher,
127 const std::vector<std::string>& test_names) override;
128 size_t RetryTests(base::TestLauncher* test_launcher,
129 const std::vector<std::string>& test_names) override;
131 private:
132 void DoRunTest(base::TestLauncher* test_launcher,
133 const std::string& test_name);
135 // Launches test named |test_name| using parallel launcher,
136 // given result of PRE_ test |pre_test_result|.
137 void RunDependentTest(base::TestLauncher* test_launcher,
138 const std::string test_name,
139 const base::TestResult& pre_test_result);
141 // Callback to receive result of a test.
142 void GTestCallback(
143 base::TestLauncher* test_launcher,
144 const std::string& test_name,
145 int exit_code,
146 const base::TimeDelta& elapsed_time,
147 bool was_timeout,
148 const std::string& output);
150 content::TestLauncherDelegate* launcher_delegate_;
152 // Store dependent test name (map is indexed by full test name).
153 typedef std::map<std::string, std::string> DependentTestMap;
154 DependentTestMap dependent_test_map_;
155 DependentTestMap reverse_dependent_test_map_;
157 // Store unique data directory prefix for test names (without PRE_ prefixes).
158 // PRE_ tests and tests that depend on them must share the same
159 // data directory. Using test name as directory name leads to too long
160 // names (exceeding UNIX_PATH_MAX, which creates a problem with
161 // process_singleton_linux). Create a randomly-named temporary directory
162 // and keep track of the names so that PRE_ tests can still re-use them.
163 typedef std::map<std::string, base::FilePath> UserDataDirMap;
164 UserDataDirMap user_data_dir_map_;
166 // Store names of all seen tests to properly handle PRE_ tests.
167 std::set<std::string> all_test_names_;
169 // Temporary directory for user data directories.
170 base::ScopedTempDir temp_dir_;
172 DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
175 bool WrapperTestLauncherDelegate::GetTests(
176 std::vector<base::TestIdentifier>* output) {
177 *output = base::GetCompiledInTests();
178 return true;
181 bool WrapperTestLauncherDelegate::ShouldRunTest(
182 const std::string& test_case_name,
183 const std::string& test_name) {
184 all_test_names_.insert(test_case_name + "." + test_name);
186 if (base::StartsWith(test_name, kManualTestPrefix,
187 base::CompareCase::SENSITIVE) &&
188 !base::CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
189 return false;
192 if (base::StartsWith(test_name, kPreTestPrefix,
193 base::CompareCase::SENSITIVE)) {
194 // We will actually run PRE_ tests, but to ensure they run on the same shard
195 // as dependent tests, handle all these details internally.
196 return false;
199 return true;
202 std::string GetPreTestName(const std::string& full_name) {
203 size_t dot_pos = full_name.find('.');
204 CHECK_NE(dot_pos, std::string::npos);
205 std::string test_case_name = full_name.substr(0, dot_pos);
206 std::string test_name = full_name.substr(dot_pos + 1);
207 return test_case_name + "." + kPreTestPrefix + test_name;
210 size_t WrapperTestLauncherDelegate::RunTests(
211 base::TestLauncher* test_launcher,
212 const std::vector<std::string>& test_names) {
213 dependent_test_map_.clear();
214 reverse_dependent_test_map_.clear();
215 user_data_dir_map_.clear();
217 // Number of additional tests to run because of dependencies.
218 size_t additional_tests_to_run_count = 0;
220 // Compute dependencies of tests to be run.
221 for (size_t i = 0; i < test_names.size(); i++) {
222 std::string full_name(test_names[i]);
223 std::string pre_test_name(GetPreTestName(full_name));
225 while (ContainsKey(all_test_names_, pre_test_name)) {
226 additional_tests_to_run_count++;
228 DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
229 dependent_test_map_[pre_test_name] = full_name;
231 DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
232 reverse_dependent_test_map_[full_name] = pre_test_name;
234 full_name = pre_test_name;
235 pre_test_name = GetPreTestName(pre_test_name);
239 for (size_t i = 0; i < test_names.size(); i++) {
240 std::string full_name(test_names[i]);
242 // Make sure no PRE_ tests were requested explicitly.
243 DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
245 if (!ContainsKey(user_data_dir_map_, full_name)) {
246 base::FilePath temp_dir;
247 CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
248 FILE_PATH_LITERAL("d"), &temp_dir));
249 user_data_dir_map_[full_name] = temp_dir;
252 // If the test has any dependencies, get to the root and start with that.
253 while (ContainsKey(reverse_dependent_test_map_, full_name))
254 full_name = GetPreTestName(full_name);
256 DoRunTest(test_launcher, full_name);
259 return test_names.size() + additional_tests_to_run_count;
262 size_t WrapperTestLauncherDelegate::RetryTests(
263 base::TestLauncher* test_launcher,
264 const std::vector<std::string>& test_names) {
265 // List of tests we can kick off right now, depending on no other tests.
266 std::vector<std::string> tests_to_run_now;
268 // We retry at least the tests requested to retry.
269 std::set<std::string> test_names_set(test_names.begin(), test_names.end());
271 // In the face of PRE_ tests, we need to retry the entire chain of tests,
272 // from the very first one.
273 for (size_t i = 0; i < test_names.size(); i++) {
274 std::string test_name(test_names[i]);
275 while (ContainsKey(reverse_dependent_test_map_, test_name)) {
276 test_name = reverse_dependent_test_map_[test_name];
277 test_names_set.insert(test_name);
281 // Discard user data directories from any previous runs. Start with
282 // fresh state.
283 for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
284 i != user_data_dir_map_.end();
285 ++i) {
286 // Delete temporary directories now to avoid using too much space in /tmp.
287 if (!base::DeleteFile(i->second, true)) {
288 LOG(WARNING) << "Failed to delete " << i->second.value();
291 user_data_dir_map_.clear();
293 for (std::set<std::string>::const_iterator i = test_names_set.begin();
294 i != test_names_set.end();
295 ++i) {
296 std::string full_name(*i);
298 // Make sure PRE_ tests and tests that depend on them share the same
299 // data directory - based it on the test name without prefixes.
300 std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
301 if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
302 base::FilePath temp_dir;
303 CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
304 FILE_PATH_LITERAL("d"), &temp_dir));
305 user_data_dir_map_[test_name_no_pre] = temp_dir;
308 size_t dot_pos = full_name.find('.');
309 CHECK_NE(dot_pos, std::string::npos);
310 std::string test_case_name = full_name.substr(0, dot_pos);
311 std::string test_name = full_name.substr(dot_pos + 1);
312 std::string pre_test_name(
313 test_case_name + "." + kPreTestPrefix + test_name);
314 if (!ContainsKey(test_names_set, pre_test_name))
315 tests_to_run_now.push_back(full_name);
318 for (size_t i = 0; i < tests_to_run_now.size(); i++)
319 DoRunTest(test_launcher, tests_to_run_now[i]);
321 return test_names_set.size();
324 void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
325 const std::string& test_name) {
326 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
328 base::CommandLine cmd_line(*base::CommandLine::ForCurrentProcess());
329 CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
330 &cmd_line, user_data_dir_map_[test_name_no_pre]));
332 base::CommandLine new_cmd_line(cmd_line.GetProgram());
333 base::CommandLine::SwitchMap switches = cmd_line.GetSwitches();
335 // Strip out gtest_output flag because otherwise we would overwrite results
336 // of the other tests.
337 switches.erase(base::kGTestOutputFlag);
339 for (base::CommandLine::SwitchMap::const_iterator iter = switches.begin();
340 iter != switches.end(); ++iter) {
341 new_cmd_line.AppendSwitchNative(iter->first, iter->second);
344 // Always enable disabled tests. This method is not called with disabled
345 // tests unless this flag was specified to the browser test executable.
346 new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
347 new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
348 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
350 char* browser_wrapper = getenv("BROWSER_WRAPPER");
352 test_launcher->LaunchChildGTestProcess(
353 new_cmd_line,
354 browser_wrapper ? browser_wrapper : std::string(),
355 TestTimeouts::action_max_timeout(),
356 base::TestLauncher::USE_JOB_OBJECTS |
357 base::TestLauncher::ALLOW_BREAKAWAY_FROM_JOB,
358 base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
359 base::Unretained(this),
360 test_launcher,
361 test_name));
364 void WrapperTestLauncherDelegate::RunDependentTest(
365 base::TestLauncher* test_launcher,
366 const std::string test_name,
367 const base::TestResult& pre_test_result) {
368 if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
369 // Only run the dependent test if PRE_ test succeeded.
370 DoRunTest(test_launcher, test_name);
371 } else {
372 // Otherwise skip the test.
373 base::TestResult test_result;
374 test_result.full_name = test_name;
375 test_result.status = base::TestResult::TEST_SKIPPED;
376 test_launcher->OnTestFinished(test_result);
378 if (ContainsKey(dependent_test_map_, test_name)) {
379 RunDependentTest(test_launcher,
380 dependent_test_map_[test_name],
381 test_result);
386 void WrapperTestLauncherDelegate::GTestCallback(
387 base::TestLauncher* test_launcher,
388 const std::string& test_name,
389 int exit_code,
390 const base::TimeDelta& elapsed_time,
391 bool was_timeout,
392 const std::string& output) {
393 base::TestResult result;
394 result.full_name = test_name;
396 // TODO(phajdan.jr): Recognize crashes.
397 if (exit_code == 0)
398 result.status = base::TestResult::TEST_SUCCESS;
399 else if (was_timeout)
400 result.status = base::TestResult::TEST_TIMEOUT;
401 else
402 result.status = base::TestResult::TEST_FAILURE;
404 result.elapsed_time = elapsed_time;
406 result.output_snippet = GetTestOutputSnippet(result, output);
408 if (ContainsKey(dependent_test_map_, test_name)) {
409 RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
410 } else {
411 // No other tests depend on this, we can delete the temporary directory now.
412 // Do so to avoid too many temporary files using lots of disk space.
413 std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
414 if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
415 if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
416 LOG(WARNING) << "Failed to delete "
417 << user_data_dir_map_[test_name_no_pre].value();
419 user_data_dir_map_.erase(test_name_no_pre);
423 test_launcher->OnTestFinished(result);
426 } // namespace
428 const char kHelpFlag[] = "help";
430 const char kLaunchAsBrowser[] = "as-browser";
432 // See kManualTestPrefix above.
433 const char kRunManualTestsFlag[] = "run-manual";
435 const char kSingleProcessTestsFlag[] = "single_process";
438 TestLauncherDelegate::~TestLauncherDelegate() {
441 int LaunchTests(TestLauncherDelegate* launcher_delegate,
442 int default_jobs,
443 int argc,
444 char** argv) {
445 DCHECK(!g_launcher_delegate);
446 g_launcher_delegate = launcher_delegate;
448 base::CommandLine::Init(argc, argv);
449 const base::CommandLine* command_line =
450 base::CommandLine::ForCurrentProcess();
452 if (command_line->HasSwitch(kHelpFlag)) {
453 PrintUsage();
454 return 0;
457 scoped_ptr<ContentMainDelegate> chrome_main_delegate(
458 launcher_delegate->CreateContentMainDelegate());
459 ContentMainParams params(chrome_main_delegate.get());
461 #if defined(OS_WIN)
462 sandbox::SandboxInterfaceInfo sandbox_info = {0};
463 InitializeSandboxInfo(&sandbox_info);
465 params.instance = GetModuleHandle(NULL);
466 params.sandbox_info = &sandbox_info;
467 #elif !defined(OS_ANDROID)
468 params.argc = argc;
469 params.argv = const_cast<const char**>(argv);
470 #endif // defined(OS_WIN)
472 if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
473 (command_line->HasSwitch(switches::kSingleProcess) &&
474 command_line->HasSwitch(base::kGTestFilterFlag)) ||
475 command_line->HasSwitch(base::kGTestListTestsFlag) ||
476 command_line->HasSwitch(base::kGTestHelpFlag)) {
477 g_params = &params;
478 return launcher_delegate->RunTestSuite(argc, argv);
481 #if !defined(OS_ANDROID)
482 if (command_line->HasSwitch(switches::kProcessType) ||
483 command_line->HasSwitch(kLaunchAsBrowser)) {
484 return ContentMain(params);
486 #endif
488 base::AtExitManager at_exit;
489 testing::InitGoogleTest(&argc, argv);
490 TestTimeouts::Initialize();
492 fprintf(stdout,
493 "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
494 "For debugging a test inside a debugger, use the\n"
495 "--gtest_filter=<your_test_name> flag along with either\n"
496 "--single_process (to run the test in one launcher/browser process) or\n"
497 "--single-process (to do the above, and also run Chrome in single-"
498 "process mode).\n");
500 base::MessageLoopForIO message_loop;
502 // Allow the |launcher_delegate| to modify |default_jobs|.
503 launcher_delegate->AdjustDefaultParallelJobs(&default_jobs);
505 WrapperTestLauncherDelegate delegate(launcher_delegate);
506 base::TestLauncher launcher(&delegate, default_jobs);
507 return (launcher.Run() ? 0 : 1);
510 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
511 return g_launcher_delegate;
514 ContentMainParams* GetContentMainParams() {
515 return g_params;
518 } // namespace content