1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "base/command_line.h"
6 #include "base/file_path.h"
7 #include "base/file_util.h"
8 #include "base/path_service.h"
9 #include "base/string_number_conversions.h"
10 #include "base/string_util.h"
11 #include "base/stringprintf.h"
12 #include "chrome/browser/ui/browser.h"
13 #include "chrome/browser/ui/browser_tabstrip.h"
14 #include "chrome/browser/ui/browser_window.h"
15 #include "chrome/browser/ui/window_snapshot/window_snapshot.h"
16 #include "chrome/common/chrome_paths.h"
17 #include "chrome/common/chrome_switches.h"
18 #include "chrome/common/chrome_version_info.h"
19 #include "chrome/test/base/in_process_browser_test.h"
20 #include "chrome/test/base/test_launcher_utils.h"
21 #include "chrome/test/base/tracing.h"
22 #include "chrome/test/base/ui_test_utils.h"
23 #include "content/public/browser/render_view_host.h"
24 #include "content/public/browser/web_contents.h"
25 #include "content/public/common/content_switches.h"
26 #include "content/public/test/browser_test_utils.h"
27 #include "googleurl/src/gurl.h"
28 #include "net/base/net_util.h"
29 #include "testing/gtest/include/gtest/gtest.h"
30 #include "third_party/skia/include/core/SkBitmap.h"
31 #include "third_party/skia/include/core/SkColor.h"
32 #include "ui/compositor/compositor_setup.h"
33 #include "ui/gfx/codec/png_codec.h"
34 #include "ui/gfx/size.h"
35 #include "ui/gl/gl_switches.h"
39 enum ReferenceImageOption
{
41 kReferenceImageCheckedIn
,
42 kReferenceImageNone
// Only check a few key pixels.
45 struct ReferencePixel
{
47 unsigned char r
, g
, b
;
50 // Command line flag for overriding the default location for putting generated
51 // test images that do not match references.
52 const char kGeneratedDir
[] = "generated-dir";
53 // Command line flag for overriding the default location for reference images.
54 const char kReferenceDir
[] = "reference-dir";
56 // Corner shadow size.
57 const int kCornerDecorationSize
= 15;
59 const int kSideDecorationSize
= 2;
61 // Reads and decodes a PNG image to a bitmap. Returns true on success. The PNG
62 // should have been encoded using |gfx::PNGCodec::Encode|.
63 bool ReadPNGFile(const FilePath
& file_path
, SkBitmap
* bitmap
) {
66 return file_util::ReadFileToString(file_path
, &png_data
) &&
67 gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&png_data
[0]),
72 // Encodes a bitmap into a PNG and write to disk. Returns true on success. The
73 // parent directory does not have to exist.
74 bool WritePNGFile(const SkBitmap
& bitmap
, const FilePath
& file_path
) {
75 std::vector
<unsigned char> png_data
;
76 if (gfx::PNGCodec::EncodeBGRASkBitmap(bitmap
, true, &png_data
) &&
77 file_util::CreateDirectory(file_path
.DirName())) {
78 int bytes_written
= file_util::WriteFile(
79 file_path
, reinterpret_cast<char*>(&png_data
[0]), png_data
.size());
80 if (bytes_written
== static_cast<int>(png_data
.size()))
86 // Write an empty file, whose name indicates the chrome revision when the ref
87 // image was generated.
88 bool WriteREVFile(const FilePath
& file_path
) {
89 if (file_util::CreateDirectory(file_path
.DirName())) {
91 int bytes_written
= file_util::WriteFile(file_path
, &one_byte
, 1);
92 if (bytes_written
== 1)
100 // Test fixture for GPU image comparison tests.
101 // TODO(kkania): Document how to add to/modify these tests.
102 class GpuPixelBrowserTest
: public InProcessBrowserTest
{
104 GpuPixelBrowserTest()
105 : ref_img_revision_(0),
106 ref_img_revision_no_older_than_(0),
107 ref_img_option_(kReferenceImageNone
) {
110 virtual void SetUpCommandLine(CommandLine
* command_line
) {
111 command_line
->AppendSwitchASCII(switches::kTestGLLib
,
115 virtual void SetUpInProcessBrowserTestFixture() {
116 InProcessBrowserTest::SetUpInProcessBrowserTestFixture();
118 CommandLine
* command_line
= CommandLine::ForCurrentProcess();
119 if (command_line
->HasSwitch(switches::kUseGpuInTests
))
120 ref_img_option_
= kReferenceImageLocal
;
122 ASSERT_TRUE(PathService::Get(chrome::DIR_TEST_DATA
, &test_data_dir_
));
123 test_data_dir_
= test_data_dir_
.AppendASCII("gpu");
125 if (command_line
->HasSwitch(kGeneratedDir
))
126 generated_img_dir_
= command_line
->GetSwitchValuePath(kGeneratedDir
);
128 generated_img_dir_
= test_data_dir_
.AppendASCII("generated");
130 switch (ref_img_option_
) {
131 case kReferenceImageLocal
:
132 if (command_line
->HasSwitch(kReferenceDir
))
133 ref_img_dir_
= command_line
->GetSwitchValuePath(kReferenceDir
);
135 ref_img_dir_
= test_data_dir_
.AppendASCII("gpu_reference");
137 case kReferenceImageCheckedIn
:
138 ref_img_dir_
= test_data_dir_
.AppendASCII("llvmpipe_reference");
144 test_name_
= testing::UnitTest::GetInstance()->current_test_info()->name();
145 const char* test_status_prefixes
[] = {"DISABLED_", "FLAKY_", "FAILS_"};
146 for (size_t i
= 0; i
< arraysize(test_status_prefixes
); ++i
) {
147 ReplaceFirstSubstringAfterOffset(
148 &test_name_
, 0, test_status_prefixes
[i
], "");
151 ui::DisableTestCompositor();
154 // If the existing ref image was saved from an revision older than the
155 // ref_img_update_revision, refresh the ref image.
156 void RunPixelTest(const gfx::Size
& tab_container_size
,
158 int64 ref_img_update_revision
,
159 const ReferencePixel
* ref_pixels
,
160 size_t ref_pixel_count
) {
161 if (ref_img_option_
== kReferenceImageLocal
) {
162 ref_img_revision_no_older_than_
= ref_img_update_revision
;
163 ObtainLocalRefImageRevision();
167 ASSERT_TRUE(tracing::BeginTracing("-test_*"));
170 browser()->window()->Activate();
172 content::DOMMessageQueue message_queue
;
173 ui_test_utils::NavigateToURL(browser(), net::FilePathToFileURL(url
));
175 // Wait for notification that page is loaded.
176 ASSERT_TRUE(message_queue
.WaitForMessage(NULL
));
177 message_queue
.ClearQueue();
179 gfx::Rect new_bounds
= GetNewTabContainerBounds(tab_container_size
);
181 std::ostringstream js_call
;
182 js_call
<< "preCallResizeInChromium(";
183 js_call
<< new_bounds
.width() << ", " << new_bounds
.height();
186 ASSERT_TRUE(content::ExecuteScript(
187 chrome::GetActiveWebContents(browser()), js_call
.str()));
190 ASSERT_TRUE(message_queue
.WaitForMessage(&message
));
191 message_queue
.ClearQueue();
192 browser()->window()->SetBounds(new_bounds
);
194 // Wait for message from test page indicating the rendering is done.
195 while (message
.compare("\"resized\"")) {
196 ASSERT_TRUE(message_queue
.WaitForMessage(&message
));
197 message_queue
.ClearQueue();
200 bool ignore_bottom_corners
= false;
201 #if defined(OS_MACOSX)
202 // On Mac Lion, bottom corners have shadows with random pixels.
203 ignore_bottom_corners
= true;
207 ASSERT_TRUE(TabSnapShotToImage(&bitmap
));
208 bool same_pixels
= true;
209 if (ref_img_option_
== kReferenceImageNone
&& ref_pixels
&& ref_pixel_count
)
210 same_pixels
= ComparePixels(bitmap
, ref_pixels
, ref_pixel_count
);
212 same_pixels
= CompareImages(bitmap
, ignore_bottom_corners
);
213 EXPECT_TRUE(same_pixels
);
216 // For debugging the flaky test, this prints out a trace of what happened on
218 std::string trace_events
;
219 ASSERT_TRUE(tracing::EndTracing(&trace_events
));
221 fprintf(stderr
, "\n\nTRACE JSON:\n\n%s\n\n", trace_events
.c_str());
225 const FilePath
& test_data_dir() const {
226 return test_data_dir_
;
230 FilePath test_data_dir_
;
231 FilePath generated_img_dir_
;
232 FilePath ref_img_dir_
;
233 int64 ref_img_revision_
;
234 // The name of the test, with any special prefixes dropped.
235 std::string test_name_
;
237 // Any local ref image generated from older revision is ignored.
238 int64 ref_img_revision_no_older_than_
;
240 // Whether use locally generated ref images, or checked in ref images, or
241 // simply check a few key pixels.
242 ReferenceImageOption ref_img_option_
;
244 // Compares the generated bitmap with the appropriate reference image on disk.
245 // Returns true iff the images were the same.
247 // If no valid reference image exists, save the generated bitmap to the disk.
248 // The image format is:
249 // <test_name>_<revision>.png
251 // WebGLTeapot_19762.png
252 // The number is the chromium revision that generated the image.
254 // On failure or on ref image generation, the image and diff image will be
255 // written to disk. The formats are:
256 // FAIL_<ref_image_name>, DIFF_<ref_image_name>
258 // FAIL_WebGLTeapot_19762.png, DIFF_WebGLTeapot_19762.png
259 bool CompareImages(const SkBitmap
& gen_bmp
, bool skip_bottom_corners
) {
260 SkBitmap ref_bmp_on_disk
;
262 FilePath img_path
= ref_img_dir_
.AppendASCII(test_name_
+ ".png");
263 bool found_ref_img
= ReadPNGFile(img_path
, &ref_bmp_on_disk
);
265 if (!found_ref_img
&& ref_img_option_
== kReferenceImageCheckedIn
) {
266 LOG(ERROR
) << "Couldn't find reference image: "
268 // No image to compare to, exit early.
272 const SkBitmap
* ref_bmp
;
273 bool save_gen
= false;
274 bool save_diff
= true;
277 if ((ref_img_revision_
<= 0 && ref_img_option_
== kReferenceImageLocal
) ||
279 chrome::VersionInfo chrome_version_info
;
280 FilePath rev_path
= ref_img_dir_
.AppendASCII(
281 test_name_
+ "_" + chrome_version_info
.LastChange() + ".rev");
282 if (!WritePNGFile(gen_bmp
, img_path
)) {
283 LOG(ERROR
) << "Can't save generated image to: "
285 << " as future reference.";
288 LOG(INFO
) << "Saved reference image to: "
292 if (!WriteREVFile(rev_path
)) {
293 LOG(ERROR
) << "Can't save revision file to: "
296 file_util::Delete(img_path
, false);
298 LOG(INFO
) << "Saved revision file to: "
302 if (ref_img_revision_
> 0) {
303 LOG(ERROR
) << "Can't read the local ref image: "
308 // If we re-generate the ref image, we save the gen and diff images so
309 // the ref image can be uploaded to the server and be viewed later.
314 ref_bmp
= &ref_bmp_on_disk
;
318 if (ref_bmp
->width() != gen_bmp
.width() ||
319 ref_bmp
->height() != gen_bmp
.height()) {
321 << "Dimensions do not match (Expected) vs (Actual):"
322 << "(" << ref_bmp
->width() << "x" << ref_bmp
->height()
324 << "(" << gen_bmp
.width() << "x" << gen_bmp
.height() << ")";
325 if (ref_img_option_
== kReferenceImageLocal
)
329 // Compare pixels and create a simple diff image.
330 int diff_pixels_count
= 0;
331 diff_bmp
.setConfig(SkBitmap::kARGB_8888_Config
,
332 gen_bmp
.width(), gen_bmp
.height());
333 diff_bmp
.allocPixels();
334 diff_bmp
.eraseColor(SK_ColorWHITE
);
335 SkAutoLockPixels
lock_bmp(gen_bmp
);
336 SkAutoLockPixels
lock_ref_bmp(*ref_bmp
);
337 SkAutoLockPixels
lock_diff_bmp(diff_bmp
);
338 // The reference images were saved with no alpha channel. Use the mask to
340 uint32_t kAlphaMask
= 0x00FFFFFF;
341 for (int x
= 0; x
< gen_bmp
.width(); ++x
) {
342 for (int y
= 0; y
< gen_bmp
.height(); ++y
) {
343 if (skip_bottom_corners
&&
344 (((x
< kCornerDecorationSize
||
345 x
>= gen_bmp
.width() - kCornerDecorationSize
) &&
346 y
>= gen_bmp
.height() - kCornerDecorationSize
) ||
347 (x
< kSideDecorationSize
||
348 x
>= gen_bmp
.width() - kSideDecorationSize
)))
350 if ((*gen_bmp
.getAddr32(x
, y
) & kAlphaMask
) !=
351 (*ref_bmp
->getAddr32(x
, y
) & kAlphaMask
)) {
353 *diff_bmp
.getAddr32(x
, y
) = 192 << 16; // red
357 if (diff_pixels_count
> 0) {
358 LOG(ERROR
) << diff_pixels_count
359 << " pixels do not match.";
360 if (ref_img_option_
== kReferenceImageLocal
) {
368 std::string ref_img_filename
= img_path
.BaseName().MaybeAsASCII();
370 FilePath img_fail_path
= generated_img_dir_
.AppendASCII(
371 "FAIL_" + ref_img_filename
);
372 if (!WritePNGFile(gen_bmp
, img_fail_path
)) {
373 LOG(ERROR
) << "Can't save generated image to: "
374 << img_fail_path
.value();
376 LOG(INFO
) << "Saved generated image to: "
377 << img_fail_path
.value();
381 FilePath img_diff_path
= generated_img_dir_
.AppendASCII(
382 "DIFF_" + ref_img_filename
);
383 if (!WritePNGFile(diff_bmp
, img_diff_path
)) {
384 LOG(ERROR
) << "Can't save generated diff image to: "
385 << img_diff_path
.value();
387 LOG(INFO
) << "Saved difference image to: "
388 << img_diff_path
.value();
394 bool ComparePixels(const SkBitmap
& gen_bmp
,
395 const ReferencePixel
* ref_pixels
,
396 size_t ref_pixel_count
) {
397 SkAutoLockPixels
lock_bmp(gen_bmp
);
399 for (size_t i
= 0; i
< ref_pixel_count
; ++i
) {
400 int x
= ref_pixels
[i
].x
;
401 int y
= ref_pixels
[i
].y
;
402 unsigned char r
= ref_pixels
[i
].r
;
403 unsigned char g
= ref_pixels
[i
].g
;
404 unsigned char b
= ref_pixels
[i
].b
;
406 DCHECK(x
>= 0 && x
< gen_bmp
.width() && y
>= 0 && y
< gen_bmp
.height());
408 unsigned char* rgba
= reinterpret_cast<unsigned char*>(
409 gen_bmp
.getAddr32(x
, y
));
411 if (rgba
[0] != b
|| rgba
[1] != g
|| rgba
[2] != r
) {
412 std::string error_message
= base::StringPrintf(
413 "pixel(%d,%d) expects [%u,%u,%u], but gets [%u,%u,%u] instead",
414 x
, y
, r
, g
, b
, rgba
[0], rgba
[1], rgba
[2]);
415 LOG(ERROR
) << error_message
.c_str();
422 // Returns a gfx::Rect representing the bounds that the browser window should
423 // have if the tab contents have the desired size.
424 gfx::Rect
GetNewTabContainerBounds(const gfx::Size
& desired_size
) {
425 gfx::Rect container_rect
;
426 chrome::GetActiveWebContents(browser())->GetContainerBounds(&container_rect
);
427 // Size cannot be negative, so use a point.
428 gfx::Point
correction(
429 desired_size
.width() - container_rect
.size().width(),
430 desired_size
.height() - container_rect
.size().height());
432 gfx::Rect window_rect
= browser()->window()->GetRestoredBounds();
433 gfx::Size new_size
= window_rect
.size();
434 new_size
.Enlarge(correction
.x(), correction
.y());
435 window_rect
.set_size(new_size
);
439 // Take snapshot of the current tab, encode it as PNG, and save to a SkBitmap.
440 bool TabSnapShotToImage(SkBitmap
* bitmap
) {
442 std::vector
<unsigned char> png
;
444 gfx::Rect root_bounds
= browser()->window()->GetBounds();
445 gfx::Rect tab_contents_bounds
;
446 chrome::GetActiveWebContents(browser())->GetContainerBounds(
447 &tab_contents_bounds
);
449 gfx::Rect
snapshot_bounds(tab_contents_bounds
.x() - root_bounds
.x(),
450 tab_contents_bounds
.y() - root_bounds
.y(),
451 tab_contents_bounds
.width(),
452 tab_contents_bounds
.height());
454 gfx::NativeWindow native_window
= browser()->window()->GetNativeWindow();
455 if (!chrome::GrabWindowSnapshotForUser(native_window
, &png
,
457 LOG(ERROR
) << "browser::GrabWindowSnapShot() failed";
461 if (!gfx::PNGCodec::Decode(reinterpret_cast<unsigned char*>(&*png
.begin()),
462 png
.size(), bitmap
)) {
463 LOG(ERROR
) << "Decode PNG to a SkBitmap failed";
469 // If no valid local revision file is located, the ref_img_revision_ is 0.
470 void ObtainLocalRefImageRevision() {
472 filter
= filter
.AppendASCII(test_name_
+ "_*.rev");
473 file_util::FileEnumerator
locator(ref_img_dir_
,
474 false, // non recursive
475 file_util::FileEnumerator::FILES
,
477 int64 max_revision
= 0;
478 std::vector
<FilePath
> outdated_revs
;
479 for (FilePath full_path
= locator
.Next();
481 full_path
= locator
.Next()) {
482 std::string filename
=
483 full_path
.BaseName().RemoveExtension().MaybeAsASCII();
484 std::string revision_string
=
485 filename
.substr(test_name_
.length() + 1);
487 bool converted
= base::StringToInt64(revision_string
, &revision
);
490 if (revision
< ref_img_revision_no_older_than_
||
491 revision
< max_revision
) {
492 outdated_revs
.push_back(full_path
);
495 max_revision
= revision
;
497 ref_img_revision_
= max_revision
;
498 for (size_t i
= 0; i
< outdated_revs
.size(); ++i
)
499 file_util::Delete(outdated_revs
[i
], false);
502 DISALLOW_COPY_AND_ASSIGN(GpuPixelBrowserTest
);
505 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest
, WebGLGreenTriangle
) {
506 // If test baseline needs to be updated after a given revision, update the
507 // following number. If no revision requirement, then 0.
508 const int64 ref_img_revision_update
= 123489;
510 const ReferencePixel ref_pixels
[] = {
513 {100, 100, 0, 255, 0},
515 {50, 150, 0, 255, 0},
516 {100, 150, 0, 255, 0},
517 {150, 150, 0, 255, 0}
519 const size_t ref_pixel_count
= sizeof(ref_pixels
) / sizeof(ReferencePixel
);
521 gfx::Size
container_size(400, 300);
523 test_data_dir().AppendASCII("pixel_webgl.html");
524 RunPixelTest(container_size
, url
, ref_img_revision_update
,
525 ref_pixels
, ref_pixel_count
);
528 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest
, CSS3DBlueBox
) {
529 // If test baseline needs to be updated after a given revision, update the
530 // following number. If no revision requirement, then 0.
531 const int64 ref_img_revision_update
= 123489;
533 const ReferencePixel ref_pixels
[] = {
538 {150, 90, 0, 0, 255},
539 {70, 125, 0, 0, 255},
542 const size_t ref_pixel_count
= sizeof(ref_pixels
) / sizeof(ReferencePixel
);
544 gfx::Size
container_size(400, 300);
546 test_data_dir().AppendASCII("pixel_css3d.html");
547 RunPixelTest(container_size
, url
, ref_img_revision_update
,
548 ref_pixels
, ref_pixel_count
);
551 IN_PROC_BROWSER_TEST_F(GpuPixelBrowserTest
, Canvas2DRedBoxHD
) {
552 // If test baseline needs to be updated after a given revision, update the
553 // following number. If no revision requirement, then 0.
554 const int64 ref_img_revision_update
= 123489;
556 const ReferencePixel ref_pixels
[] = {
559 {60, 100, 127, 0, 0},
560 {140, 100, 127, 0, 0},
563 const size_t ref_pixel_count
= sizeof(ref_pixels
) / sizeof(ReferencePixel
);
565 gfx::Size
container_size(400, 300);
567 test_data_dir().AppendASCII("pixel_canvas2d.html");
568 RunPixelTest(container_size
, url
, ref_img_revision_update
,
569 ref_pixels
, ref_pixel_count
);
572 class Canvas2DPixelTestSD
: public GpuPixelBrowserTest
{
574 virtual void SetUpCommandLine(CommandLine
* command_line
) {
575 GpuPixelBrowserTest::SetUpCommandLine(command_line
);
576 command_line
->AppendSwitch(switches::kDisableAccelerated2dCanvas
);
580 IN_PROC_BROWSER_TEST_F(Canvas2DPixelTestSD
, Canvas2DRedBoxSD
) {
581 // If test baseline needs to be updated after a given revision, update the
582 // following number. If no revision requirement, then 0.
583 const int64 ref_img_revision_update
= 123489;
585 const ReferencePixel ref_pixels
[] = {
588 {60, 100, 127, 0, 0},
589 {140, 100, 127, 0, 0},
592 const size_t ref_pixel_count
= sizeof(ref_pixels
) / sizeof(ReferencePixel
);
594 gfx::Size
container_size(400, 300);
596 test_data_dir().AppendASCII("pixel_canvas2d.html");
597 RunPixelTest(container_size
, url
, ref_img_revision_update
,
598 ref_pixels
, ref_pixel_count
);