1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef PPAPI_TESTS_TEST_CASE_H_
6 #define PPAPI_TESTS_TEST_CASE_H_
15 #include "ppapi/c/pp_resource.h"
16 #include "ppapi/c/pp_time.h"
17 #include "ppapi/c/private/ppb_testing_private.h"
18 #include "ppapi/cpp/message_loop.h"
19 #include "ppapi/cpp/view.h"
20 #include "ppapi/tests/test_utils.h"
21 #include "ppapi/tests/testing_instance.h"
23 #if (defined __native_client__)
24 #include "ppapi/cpp/var.h"
26 #include "ppapi/cpp/private/var_private.h"
29 class TestingInstance
;
32 namespace deprecated
{
33 class ScriptableObject
;
37 // Individual classes of tests derive from this generic test case.
40 explicit TestCase(TestingInstance
* instance
);
43 // Optionally override to do testcase specific initialization.
44 // Default implementation just returns true.
47 // Override to implement the test case. It will be called after the plugin is
48 // first displayed, passing a string. If the string is empty, RunTests should
49 // run all tests for this test case. Otherwise, it must be a comma-delimited
50 // list of test names, possibly prefixed. E.g.:
51 // "Foo_GoodTest,DISABLED_Foo_BadTest,Foo_OtherGoodTest"
52 // All listed tests which are not prefixed will be run.
54 // This should generally be implemented in a TestCase subclass using the
56 virtual void RunTests(const std::string
& test_filter
) = 0;
58 static std::string
MakeFailureMessage(const char* file
, int line
,
61 #if !(defined __native_client__)
62 // Returns the scriptable test object for the current test, if any.
63 // Internally, this uses CreateTestObject which each test overrides.
64 pp::VarPrivate
GetTestObject();
65 void ResetTestObject() { test_object_
= pp::VarPrivate(); }
68 // A function that is invoked whenever HandleMessage is called on the
69 // associated TestingInstance. Default implementation does nothing. TestCases
70 // that want to handle incoming postMessage events should override this
72 virtual void HandleMessage(const pp::Var
& message_data
);
74 // A function that is invoked whenever DidChangeView is called on the
75 // associated TestingInstance. Default implementation does nothing. TestCases
76 // that want to handle view changes should override this method.
77 virtual void DidChangeView(const pp::View
& view
);
79 // A function that is invoked whenever HandleInputEvent is called on the
80 // associated TestingInstance. Default implementation returns false. TestCases
81 // that want to handle view changes should override this method.
82 virtual bool HandleInputEvent(const pp::InputEvent
& event
);
84 void IgnoreLeakedVar(int64_t id
);
86 TestingInstance
* instance() { return instance_
; }
88 const PPB_Testing_Private
* testing_interface() { return testing_interface_
; }
90 static void QuitMainMessageLoop(PP_Instance instance
);
92 const std::map
<std::string
, bool>& remaining_tests() {
93 return remaining_tests_
;
95 const std::set
<std::string
>& skipped_tests() {
96 return skipped_tests_
;
100 #if !(defined __native_client__)
101 // Overridden by each test to supply a ScriptableObject corresponding to the
102 // test. There can only be one object created for all tests in a given class,
103 // so be sure your object is designed to be re-used.
105 // This object should be created on the heap. Ownership will be passed to the
106 // caller. Return NULL if there is no supported test object (the default).
107 virtual pp::deprecated::ScriptableObject
* CreateTestObject();
110 // Checks whether the testing interface is available. Returns true if it is,
111 // false otherwise. If it is not available, adds a descriptive error. This is
112 // for use by tests that require the testing interface.
113 bool CheckTestingInterface();
115 // Makes sure the test is run over HTTP.
116 bool EnsureRunningOverHTTP();
118 // Returns true if |filter| only contains a TestCase name, which normally
119 // means "run all tests". Some TestCases require special setup for individual
120 // tests, and can use this function to decide whether to ignore those tests.
121 bool ShouldRunAllTests(const std::string
& filter
);
123 // Return true if the given test name matches the filter. This is true if
124 // (a) filter is empty or (b) test_name matches a test name listed in filter
126 bool ShouldRunTest(const std::string
& test_name
, const std::string
& filter
);
128 // Check for leaked resources and vars at the end of the test. If any exist,
129 // return a string with some information about the error. Otherwise, return
132 // You should pass the error string from the test so far; if it is non-empty,
133 // CheckResourcesAndVars will do nothing and return the same string.
134 std::string
CheckResourcesAndVars(std::string errors
);
136 PP_TimeTicks
NowInTimeTicks();
138 // Run the given test method on a background thread and return the result.
140 std::string
RunOnThread(std::string(T::*test_to_run
)()) {
141 if (!testing_interface_
) {
142 return "Testing blocking callbacks requires the testing interface. In "
143 "Chrome, use the --enable-pepper-testing flag.";
145 // These tests are only valid if running out-of-process (threading is not
146 // supported in-process). For in-process, just consider it a pass.
147 if (!testing_interface_
->IsOutOfProcess())
148 return std::string();
149 pp::MessageLoop
background_loop(instance_
);
150 ThreadedTestRunner
<T
> runner(instance_
->pp_instance(),
151 static_cast<T
*>(this), test_to_run
, background_loop
);
152 RunOnThreadInternal(&ThreadedTestRunner
<T
>::ThreadFunction
, &runner
,
154 return runner
.result();
157 // Pointer to the instance that owns us.
158 TestingInstance
* instance_
;
160 // NULL unless InitTestingInterface is called.
161 const PPB_Testing_Private
* testing_interface_
;
163 void set_callback_type(CallbackType callback_type
) {
164 callback_type_
= callback_type
;
166 CallbackType
callback_type() const {
167 return callback_type_
;
172 class ThreadedTestRunner
{
174 typedef std::string(T::*TestMethodType
)();
175 ThreadedTestRunner(PP_Instance instance
,
177 TestMethodType test_to_run
,
178 pp::MessageLoop loop
)
179 : instance_(instance
),
180 test_case_(test_case
),
181 test_to_run_(test_to_run
),
184 const std::string
& result() { return result_
; }
185 static void ThreadFunction(void* runner
) {
186 static_cast<ThreadedTestRunner
<T
>*>(runner
)->Run();
191 int32_t result
= loop_
.AttachToCurrentThread();
192 static_cast<void>(result
); // result is not used in the RELEASE build.
193 PP_DCHECK(PP_OK
== result
);
194 result_
= (test_case_
->*test_to_run_
)();
195 // Now give the loop a chance to clean up.
196 loop_
.PostQuit(true /* should_destroy */);
198 // Tell the main thread to quit its nested message loop, now that the test
200 TestCase::QuitMainMessageLoop(instance_
);
204 PP_Instance instance_
;
206 TestMethodType test_to_run_
;
207 pp::MessageLoop loop_
;
210 // The internals for RunOnThread. This allows us to avoid including
211 // pp_thread.h in this header file, since it includes system headers like
213 // RunOnThreadInternal launches a new thread to run |thread_func|, waits
214 // for it to complete using RunMessageLoop(), then joins.
215 void RunOnThreadInternal(void (*thread_func
)(void*),
217 const PPB_Testing_Private
* testing_interface
);
219 static void DoQuitMainMessageLoop(void* pp_instance
, int32_t result
);
221 // Passed when creating completion callbacks in some tests. This determines
222 // what kind of callback we use for the test.
223 CallbackType callback_type_
;
225 // Var ids that should be ignored when checking for leaks on shutdown.
226 std::set
<int64_t> ignored_leaked_vars_
;
228 // The tests that were found in test_filter. The bool indicates whether the
229 // test should be run (i.e., it will be false if the test name was prefixed in
230 // the test_filter string).
232 // This is initialized lazily the first time that ShouldRunTest is called.
233 std::map
<std::string
, bool> filter_tests_
;
234 // Flag indicating whether we have populated filter_tests_ yet.
235 bool have_populated_filter_tests_
;
236 // This is initialized with the contents of filter_tests_. As each test is
237 // run, it is removed from remaining_tests_. When RunTests is finished,
238 // remaining_tests_ should be empty. Any remaining tests are tests that were
239 // listed in the test_filter but didn't match any calls to ShouldRunTest,
240 // meaning it was probably a typo. TestingInstance should log this and
241 // consider it a failure.
242 std::map
<std::string
, bool> remaining_tests_
;
244 // If ShouldRunTest is called but the given test name doesn't match anything
245 // in the test_filter, the test name will be added here. This allows
246 // TestingInstance to detect when not all tests were listed.
247 std::set
<std::string
> skipped_tests_
;
249 #if !(defined __native_client__)
250 // Holds the test object, if any was retrieved from CreateTestObject.
251 pp::VarPrivate test_object_
;
255 // This class is an implementation detail.
256 class TestCaseFactory
{
258 typedef TestCase
* (*Method
)(TestingInstance
* instance
);
260 TestCaseFactory(const char* name
, Method method
)
268 friend class TestingInstance
;
270 TestCaseFactory
* next_
;
274 static TestCaseFactory
* head_
;
279 // The internal namespace contains implementation details that are used by
280 // the ASSERT macros.
282 // This base class provides a ToString that works for classes that can be
283 // converted to a string using std::stringstream. Later, we'll do
284 // specializations for types that we know will work with this approach.
286 struct StringinatorBase
{
287 static std::string
ToString(const T
& value
) {
288 std::stringstream stream
;
293 // Not implemented, do not use.
294 // Note, these are protected because Windows complains if I make these private
295 // and then inherit StringinatorBase (even though they're never used).
300 // This default class template is for types that we don't recognize as
301 // something we can convert into a string using stringstream. Types that we
302 // know *can* be turned to a string should have specializations below.
304 struct Stringinator
{
305 static std::string
ToString(const T
& value
) {
306 return std::string();
309 // Not implemented, do not use.
314 // Define some full specializations for types that can just use stringstream.
315 #define DEFINE_STRINGINATOR_FOR_TYPE(type) \
317 struct Stringinator<type> : public StringinatorBase<type> {};
318 DEFINE_STRINGINATOR_FOR_TYPE(int32_t);
319 DEFINE_STRINGINATOR_FOR_TYPE(uint32_t);
320 DEFINE_STRINGINATOR_FOR_TYPE(int64_t);
321 DEFINE_STRINGINATOR_FOR_TYPE(uint64_t);
322 DEFINE_STRINGINATOR_FOR_TYPE(float);
323 DEFINE_STRINGINATOR_FOR_TYPE(double);
324 DEFINE_STRINGINATOR_FOR_TYPE(bool);
325 DEFINE_STRINGINATOR_FOR_TYPE(std::string
);
326 #undef DEFINE_STRINGINATOR_FOR_TYPE
329 std::string
ToString(const T
& param
) {
330 return Stringinator
<T
>::ToString(param
);
333 // This overload is necessary to allow enum values (such as those from
334 // pp_errors.h, including PP_OK) to work. They won't automatically convert to
335 // an integral type to instantiate the above function template.
336 inline std::string
ToString(int32_t param
) {
337 return Stringinator
<int32_t>::ToString(param
);
340 inline std::string
ToString(const char* c_string
) {
341 return std::string(c_string
);
344 // This overload deals with pointers.
346 std::string
ToString(const T
* ptr
) {
347 uintptr_t ptr_val
= reinterpret_cast<uintptr_t>(ptr
);
348 std::stringstream stream
;
353 // ComparisonHelper classes wrap the left-hand parameter of a binary comparison
354 // ASSERT. The correct class gets chosen based on whether or not it's a NULL or
355 // 0 literal. If it is a NULL/0 literal, we use NullLiteralComparisonHelper.
356 // For all other parameters, we use ComparisonHelper. There's also a
357 // specialization of ComparisonHelper for int below (see below for why
360 // ComparisonHelper does two things for the left param:
361 // 1) Provides all the appropriate CompareXX functions (CompareEQ, etc).
362 // 2) Provides ToString.
364 struct ComparisonHelper
{
365 explicit ComparisonHelper(const T
& param
) : value(param
) {}
367 bool CompareEQ(const U
& right
) const {
368 return value
== right
;
371 bool CompareNE(const U
& right
) const {
372 return value
!= right
;
375 bool CompareLT(const U
& right
) const {
376 return value
< right
;
379 bool CompareGT(const U
& right
) const {
380 return value
> right
;
383 bool CompareLE(const U
& right
) const {
384 return value
<= right
;
387 bool CompareGE(const U
& right
) const {
388 return value
>= right
;
390 std::string
ToString() const {
391 return internal::ToString(value
);
396 // Used for NULL or 0.
397 struct NullLiteralComparisonHelper
{
398 NullLiteralComparisonHelper() : value(0) {}
400 bool CompareEQ(const U
& right
) const {
404 bool CompareNE(const U
& right
) const {
408 bool CompareLT(const U
& right
) const {
412 bool CompareGT(const U
& right
) const {
416 bool CompareLE(const U
& right
) const {
420 bool CompareGE(const U
& right
) const {
423 std::string
ToString() const {
424 return std::string("0");
429 // This class makes it safe to use an integer literal (like 5, or 123) when
430 // comparing with an unsigned. For example:
431 // ASSERT_EQ(1, some_vector.size());
432 // We do a lot of those comparisons, so this makes it easy to get it right
433 // (rather than forcing assertions to use unsigned literals like 5u or 123u).
435 // This is slightly risky; we're static_casting an int to whatever's on the
436 // right. If the left value is negative and the right hand side is a large
437 // unsigned value, it's possible that the comparison will succeed when maybe
438 // it shouldn't have.
439 // TODO(dmichael): It should be possible to fix this and upgrade int32_t and
440 // uint32_t to int64_t for the comparison, and make any unsafe
441 // comparisons into compile errors.
443 struct ComparisonHelper
<int> {
444 explicit ComparisonHelper(int param
) : value(param
) {}
446 bool CompareEQ(const U
& right
) const {
447 return static_cast<U
>(value
) == right
;
450 bool CompareNE(const U
& right
) const {
451 return static_cast<U
>(value
) != right
;
454 bool CompareLT(const U
& right
) const {
455 return static_cast<U
>(value
) < right
;
458 bool CompareGT(const U
& right
) const {
459 return static_cast<U
>(value
) > right
;
462 bool CompareLE(const U
& right
) const {
463 return static_cast<U
>(value
) <= right
;
466 bool CompareGE(const U
& right
) const {
467 return static_cast<U
>(value
) >= right
;
469 std::string
ToString() const {
470 return internal::ToString(value
);
476 // The default is for the case there the parameter is *not* a NULL or 0 literal.
477 template <bool is_null_literal
>
478 struct ParameterWrapper
{
480 static ComparisonHelper
<T
> WrapValue(const T
& value
) {
481 return ComparisonHelper
<T
>(value
);
483 // This overload is so that we can deal with values from anonymous enums,
484 // like the one in pp_errors.h. The function template above won't be
485 // considered a match by the compiler.
486 static ComparisonHelper
<int> WrapValue(int value
) {
487 return ComparisonHelper
<int>(value
);
491 // The parameter to WrapValue *is* a NULL or 0 literal.
493 struct ParameterWrapper
<true> {
494 // We just use "..." and ignore the parameter. This sidesteps some problems we
495 // would run in to (not all compilers have the same set of constraints).
496 // - We can't use a pointer type, because int and enums won't convert.
497 // - We can't use an integral type, because pointers won't convert.
498 // - We can't overload, because it will sometimes be ambiguous.
499 // - We can't templatize and deduce the parameter. Some compilers will deduce
500 // int for NULL, and then refuse to convert NULL to an int.
502 // We know in this case that the value is 0, so there's no need to capture the
503 // value. We also know it's a fundamental type, so it's safe to pass to "...".
504 // (It's illegal to pass non-POD types to ...).
505 static NullLiteralComparisonHelper
WrapValue(...) {
506 return NullLiteralComparisonHelper();
510 // IS_NULL_LITERAL(type) is a little template metaprogramming for determining
511 // if a type is a null or zero literal (NULL or 0 or a constant that evaluates
513 // The idea is that for NULL or 0, any pointer type is always a better match
514 // than "...". But no other pointer types or literals should convert
515 // automatically to InternalDummyClass.
516 struct InternalDummyClass
{};
517 char TestNullLiteral(const InternalDummyClass
*);
518 struct BiggerThanChar
{ char dummy
[2]; };
519 BiggerThanChar
TestNullLiteral(...);
520 // If the compiler chooses the overload of TestNullLiteral which returns char,
521 // then we know the value converts automatically to InternalDummyClass*, which
522 // should only be true of NULL and 0 constants.
523 #define IS_NULL_LITERAL(a) sizeof(internal::TestNullLiteral(a)) == sizeof(char)
525 template <class T
, class U
>
526 static std::string
MakeBinaryComparisonFailureMessage(
527 const char* comparator
,
530 const char* left_precompiler_string
,
531 const char* right_precompiler_string
,
532 const char* file_name
,
534 std::string error_msg
=
535 std::string("Failed ASSERT_") + comparator
+ "(" +
536 left_precompiler_string
+ ", " + right_precompiler_string
+ ")";
537 std::string
left_string(left
.ToString());
538 std::string
right_string(ToString(right
));
539 if (!left_string
.empty())
540 error_msg
+= " Left: (" + left_string
+ ")";
542 if (!right_string
.empty())
543 error_msg
+= " Right: (" + right_string
+ ")";
545 return TestCase::MakeFailureMessage(file_name
, line_number
,
549 // The Comparison function templates allow us to pass the parameter for
550 // ASSERT macros below and have them be evaluated only once. This is important
551 // for cases where the parameter might be an expression with side-effects, like
553 #define DEFINE_COMPARE_FUNCTION(comparator_name) \
554 template <class T, class U> \
555 std::string Compare ## comparator_name ( \
558 const char* left_precompiler_string, \
559 const char* right_precompiler_string, \
560 const char* file_name, \
562 if (!(left.Compare##comparator_name(right))) { \
563 return MakeBinaryComparisonFailureMessage(#comparator_name, \
566 left_precompiler_string, \
567 right_precompiler_string, \
571 return std::string(); \
573 DEFINE_COMPARE_FUNCTION(EQ
)
574 DEFINE_COMPARE_FUNCTION(NE
)
575 DEFINE_COMPARE_FUNCTION(LT
)
576 DEFINE_COMPARE_FUNCTION(LE
)
577 DEFINE_COMPARE_FUNCTION(GT
)
578 DEFINE_COMPARE_FUNCTION(GE
)
579 #undef DEFINE_COMPARE_FUNCTION
580 inline std::string
CompareDoubleEq(ComparisonHelper
<double> left
,
582 const char* left_precompiler_string
,
583 const char* right_precompiler_string
,
584 const char* file_name
,
586 if (!(std::fabs(left
.value
- right
) <=
587 std::numeric_limits
<double>::epsilon())) {
588 return MakeBinaryComparisonFailureMessage(
589 "~=", left
, right
, left_precompiler_string
, right_precompiler_string
,
592 return std::string();
595 } // namespace internal
597 // Use the REGISTER_TEST_CASE macro in your TestCase implementation file to
598 // register your TestCase. If your test is named TestFoo, then add the
599 // following to test_foo.cc:
601 // REGISTER_TEST_CASE(Foo);
603 // This will cause your test to be included in the set of known tests.
605 #define REGISTER_TEST_CASE(name) \
606 static TestCase* Test##name##_FactoryMethod(TestingInstance* instance) { \
607 return new Test##name(instance); \
609 static TestCaseFactory g_Test##name_factory( \
610 #name, &Test##name##_FactoryMethod \
613 // Helper macro for calling functions implementing specific tests in the
614 // RunTest function. This assumes the function name is TestFoo where Foo is the
616 #define RUN_TEST(name, test_filter) \
617 if (ShouldRunTest(#name, test_filter)) { \
618 set_callback_type(PP_OPTIONAL); \
619 PP_TimeTicks start_time(NowInTimeTicks()); \
620 instance_->LogTest(#name, \
621 CheckResourcesAndVars(Test##name()), \
625 // Like RUN_TEST above but forces functions taking callbacks to complete
626 // asynchronously on success or error.
627 #define RUN_TEST_FORCEASYNC(name, test_filter) \
628 if (ShouldRunTest(#name, test_filter)) { \
629 set_callback_type(PP_REQUIRED); \
630 PP_TimeTicks start_time(NowInTimeTicks()); \
631 instance_->LogTest(#name"ForceAsync", \
632 CheckResourcesAndVars(Test##name()), \
636 #define RUN_TEST_BLOCKING(test_case, name, test_filter) \
637 if (ShouldRunTest(#name, test_filter)) { \
638 set_callback_type(PP_BLOCKING); \
639 PP_TimeTicks start_time(NowInTimeTicks()); \
640 instance_->LogTest( \
642 CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
646 #define RUN_TEST_BACKGROUND(test_case, name, test_filter) \
647 if (ShouldRunTest(#name, test_filter)) { \
648 PP_TimeTicks start_time(NowInTimeTicks()); \
649 instance_->LogTest( \
651 CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
655 #define RUN_TEST_FORCEASYNC_AND_NOT(name, test_filter) \
657 RUN_TEST_FORCEASYNC(name, test_filter); \
658 RUN_TEST(name, test_filter); \
661 // Run a test with all possible callback types.
662 #define RUN_CALLBACK_TEST(test_case, name, test_filter) \
664 RUN_TEST_FORCEASYNC(name, test_filter); \
665 RUN_TEST(name, test_filter); \
666 RUN_TEST_BLOCKING(test_case, name, test_filter); \
667 RUN_TEST_BACKGROUND(test_case, name, test_filter); \
670 #define RUN_TEST_WITH_REFERENCE_CHECK(name, test_filter) \
671 if (ShouldRunTest(#name, test_filter)) { \
672 set_callback_type(PP_OPTIONAL); \
673 uint32_t objects = testing_interface_->GetLiveObjectsForInstance( \
674 instance_->pp_instance()); \
675 std::string error_message = Test##name(); \
676 if (error_message.empty() && \
677 testing_interface_->GetLiveObjectsForInstance( \
678 instance_->pp_instance()) != objects) \
679 error_message = MakeFailureMessage(__FILE__, __LINE__, \
680 "reference leak check"); \
681 PP_TimeTicks start_time(NowInTimeTicks()); \
682 instance_->LogTest(#name, \
683 CheckResourcesAndVars(error_message), \
686 // TODO(dmichael): Add CheckResourcesAndVars above when Windows tests pass
687 // cleanly. crbug.com/173503
689 // Helper macros for checking values in tests, and returning a location
690 // description of the test fails.
691 #define ASSERT_TRUE(cmd) \
694 return MakeFailureMessage(__FILE__, __LINE__, #cmd); \
696 #define ASSERT_FALSE(cmd) ASSERT_TRUE(!(cmd))
697 #define COMPARE_BINARY_INTERNAL(comparison_type, a, b) \
698 internal::Compare##comparison_type( \
699 internal::ParameterWrapper<IS_NULL_LITERAL(a)>::WrapValue(a), \
705 #define ASSERT_BINARY_INTERNAL(comparison_type, a, b) \
707 std::string internal_assert_result_string = \
708 COMPARE_BINARY_INTERNAL(comparison_type, a, b); \
709 if (!internal_assert_result_string.empty()) { \
710 return internal_assert_result_string; \
713 #define ASSERT_EQ(a, b) ASSERT_BINARY_INTERNAL(EQ, a, b)
714 #define ASSERT_NE(a, b) ASSERT_BINARY_INTERNAL(NE, a, b)
715 #define ASSERT_LT(a, b) ASSERT_BINARY_INTERNAL(LT, a, b)
716 #define ASSERT_LE(a, b) ASSERT_BINARY_INTERNAL(LE, a, b)
717 #define ASSERT_GT(a, b) ASSERT_BINARY_INTERNAL(GT, a, b)
718 #define ASSERT_GE(a, b) ASSERT_BINARY_INTERNAL(GE, a, b)
719 #define ASSERT_DOUBLE_EQ(a, b) \
721 std::string internal_assert_result_string = \
722 internal::CompareDoubleEq( \
723 internal::ParameterWrapper<IS_NULL_LITERAL(a)>::WrapValue(a), \
729 if (!internal_assert_result_string.empty()) { \
730 return internal_assert_result_string; \
733 // Runs |function| as a subtest and asserts that it has passed.
734 #define ASSERT_SUBTEST_SUCCESS(function) \
736 std::string result = (function); \
737 if (!result.empty()) \
738 return TestCase::MakeFailureMessage(__FILE__, __LINE__, result.c_str()); \
741 #define PASS() return std::string()
743 #endif // PPAPI_TESTS_TEST_CASE_H_