cairo/test/cairo-test.c

   1 /*
   2  * Copyright © 2004 Red Hat, Inc.
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software
   5  * and its documentation for any purpose is hereby granted without
   6  * fee, provided that the above copyright notice appear in all copies
   7  * and that both that copyright notice and this permission notice
   8  * appear in supporting documentation, and that the name of
   9  * Red Hat, Inc. not be used in advertising or publicity pertaining to
  10  * distribution of the software without specific, written prior
  11  * permission. Red Hat, Inc. makes no representations about the
  12  * suitability of this software for any purpose.  It is provided "as
  13  * is" without express or implied warranty.
  14  *
  15  * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  16  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  17  * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
  18  * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  19  * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  20  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  21  * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22  *
  23  * Author: Carl D. Worth <cworth@cworth.org>
  24  */
  25
  26 #define _GNU_SOURCE 1   /* for feenableexcept() et al */
  27 #define _POSIX_C_SOURCE 2000112L /* for flockfile() et al */
  28
  29 #if HAVE_CONFIG_H
  30 #include "config.h"
  31 #endif
  32
  33 #include <stdio.h>
  34 #include <stdlib.h>
  35 #include <stdarg.h>
  36 #include <ctype.h>
  37 #if HAVE_FEENABLEEXCEPT
  38 #include <fenv.h>
  39 #endif
  40 #include <assert.h>
  41 #if HAVE_UNISTD_H
  42 #include <unistd.h>
  43 #endif
  44 #include <errno.h>
  45 #include <string.h>
  46 #if HAVE_FCFINI
  47 #include <fontconfig/fontconfig.h>
  48 #endif
  49 #if HAVE_PTHREAD_H
  50 #include <pthread.h>
  51 #endif
  52 #if HAVE_SYS_STAT_H
  53 #include <sys/stat.h>
  54 #endif
  55
  56 #if HAVE_VALGRIND
  57 #include <valgrind.h>
  58 #else
  59 #define RUNNING_ON_VALGRIND 0
  60 #endif
  61
  62 #if HAVE_MEMFAULT
  63 #include <memfault.h>
  64 #define MF(x) x
  65 #else
  66 #define MF(x)
  67 #endif
  68
  69 #include "cairo-test.h"
  70
  71 #include "buffer-diff.h"
  72
  73 #ifdef _MSC_VER
  74 #include <crtdbg.h>
  75 #define vsnprintf _vsnprintf
  76 #define access _access
  77 #define F_OK 0
  78 #endif
  79 #ifndef FALSE
  80 #define FALSE 0
  81 #endif
  82 #ifndef TRUE
  83 #define TRUE !FALSE
  84 #endif
  85
  86 static cairo_user_data_key_t _cairo_test_context_key;
  87
  88 static void
  89 _xunlink (const cairo_test_context_t *ctx, const char *pathname);
  90
  91 static const char *fail_face = "", *normal_face = "";
  92
  93 #define CAIRO_TEST_LOG_SUFFIX ".log"
  94 #define CAIRO_TEST_PNG_SUFFIX "-out.png"
  95 #define CAIRO_TEST_REF_SUFFIX "-ref.png"
  96 #define CAIRO_TEST_DIFF_SUFFIX "-diff.png"
  97
  98 #define NUM_DEVICE_OFFSETS 1
  99
 100 static const char *vector_ignored_tests[] = {
 101     /* We can't match the results of tests that depend on
 102      * CAIRO_ANTIALIAS_NONE/SUBPIXEL for vector backends
 103      * (nor do we care). */
 104     /* XXX Perhaps this should be moved to a flag in cairo_test_t? */
 105     "a1-image-sample",
 106     "a1-traps-sample",
 107     "ft-text-antialias-none",
 108     "rectangle-rounding-error",
 109     "text-antialias-gray",
 110     "text-antialias-none",
 111     "text-antialias-subpixel",
 112     "text-lcd-filter-fir3",
 113     "text-lcd-filter-fir5",
 114     "text-lcd-filter-intra-pixel",
 115     "text-lcd-filter-none",
 116     "unantialiased-shapes",
 117
 118     /* Nor do we care about rendering anomalies in external renderers. */
 119     "fill-degenerate-sort-order",
 120     NULL
 121 };
 122
 123 static void
 124 _cairo_test_init (cairo_test_context_t *ctx,
 125                   const cairo_test_t *test,
 126                   const char *test_name,
 127                   cairo_test_status_t expectation)
 128 {
 129     char *log_name;
 130
 131     MF (VALGRIND_DISABLE_FAULTS ());
 132
 133 #if HAVE_FEENABLEEXCEPT
 134     feenableexcept (FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
 135 #endif
 136
 137     ctx->test = test;
 138     ctx->test_name = test_name;
 139     ctx->expectation = expectation;
 140
 141     ctx->malloc_failure = 0;
 142 #if HAVE_MEMFAULT
 143     if (getenv ("CAIRO_TEST_MALLOC_FAILURE"))
 144         ctx->malloc_failure = atoi (getenv ("CAIRO_TEST_MALLOC_FAILURE"));
 145     if (ctx->malloc_failure && ! RUNNING_ON_MEMFAULT ())
 146         ctx->malloc_failure = 0;
 147 #endif
 148
 149     xasprintf (&log_name, "%s%s", test_name, CAIRO_TEST_LOG_SUFFIX);
 150     _xunlink (NULL, log_name);
 151
 152     ctx->log_file = fopen (log_name, "a");
 153     if (ctx->log_file == NULL) {
 154         fprintf (stderr, "Error opening log file: %s\n", log_name);
 155         ctx->log_file = stderr;
 156     }
 157     free (log_name);
 158
 159     ctx->srcdir = getenv ("srcdir");
 160     if (ctx->srcdir == NULL)
 161         ctx->srcdir = ".";
 162
 163     ctx->refdir = getenv ("CAIRO_REF_DIR");
 164
 165     ctx->ref_name = NULL;
 166     ctx->ref_image = NULL;
 167     ctx->ref_image_flattened = NULL;
 168
 169     ctx->thread = 0;
 170
 171     {
 172         int tmp_num_targets;
 173         cairo_bool_t tmp_limited_targets;
 174         ctx->targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
 175         ctx->num_targets = tmp_num_targets;
 176         ctx->limited_targets = tmp_limited_targets;
 177     }
 178
 179     printf ("\nTESTING %s\n", test_name);
 180 }
 181
 182 void
 183 cairo_test_init (cairo_test_context_t *ctx,
 184                  const char *test_name)
 185 {
 186     _cairo_test_init (ctx, NULL, test_name, CAIRO_TEST_SUCCESS);
 187 }
 188
 189 static void
 190 cairo_test_init_thread (cairo_test_context_t *ctx,
 191                         cairo_test_context_t *master,
 192                         int thread)
 193 {
 194     MF (VALGRIND_DISABLE_FAULTS ());
 195
 196     *ctx = *master;
 197     ctx->thread = thread;
 198 }
 199
 200 void
 201 cairo_test_fini (cairo_test_context_t *ctx)
 202 {
 203     if (ctx->thread != 0)
 204         return;
 205
 206     if (ctx->log_file == NULL)
 207         return;
 208
 209     if (ctx->log_file != stderr)
 210         fclose (ctx->log_file);
 211     ctx->log_file = NULL;
 212
 213     if (ctx->ref_name != NULL)
 214         free (ctx->ref_name);
 215     cairo_surface_destroy (ctx->ref_image);
 216     cairo_surface_destroy (ctx->ref_image_flattened);
 217
 218     cairo_boilerplate_free_targets (ctx->targets_to_test);
 219
 220     cairo_debug_reset_static_data ();
 221 #if HAVE_FCFINI
 222     FcFini ();
 223 #endif
 224 }
 225
 226 void
 227 cairo_test_log (const cairo_test_context_t *ctx, const char *fmt, ...)
 228 {
 229     va_list va;
 230     FILE *file = ctx && ctx->log_file ? ctx->log_file : stderr;
 231
 232     va_start (va, fmt);
 233     vfprintf (file, fmt, va);
 234     va_end (va);
 235 }
 236
 237 void
 238 cairo_test_log_path (const cairo_test_context_t *ctx,
 239                      const cairo_path_t *path)
 240 {
 241   int i;
 242
 243   for (i = 0; i < path->num_data; i += path->data[i].header.length) {
 244     cairo_path_data_t *data = &path->data[i];
 245     switch (data->header.type) {
 246     case CAIRO_PATH_MOVE_TO:
 247         cairo_test_log (ctx,
 248                         "    cairo_move_to (cr, %g, %g);\n",
 249                         data[1].point.x, data[1].point.y);
 250         break;
 251     case CAIRO_PATH_LINE_TO:
 252         cairo_test_log (ctx,
 253                         "    cairo_line_to (cr, %g, %g);\n",
 254                         data[1].point.x, data[1].point.y);
 255         break;
 256     case CAIRO_PATH_CURVE_TO:
 257         cairo_test_log (ctx,
 258                         "    cairo_curve_to (cr, %g, %g, %g, %g, %g, %g);\n",
 259                         data[1].point.x, data[1].point.y,
 260                         data[2].point.x, data[2].point.y,
 261                         data[3].point.x, data[3].point.y);
 262         break;
 263     case CAIRO_PATH_CLOSE_PATH:
 264         cairo_test_log (ctx,
 265                         "    cairo_close_path (cr);\n\n");
 266         break;
 267     default:
 268         assert (0);
 269     }
 270   }
 271 }
 272
 273 static void
 274 _xunlink (const cairo_test_context_t *ctx, const char *pathname)
 275 {
 276     if (unlink (pathname) < 0 && errno != ENOENT) {
 277         cairo_test_log (ctx, "Error: Cannot remove %s: %s\n",
 278                         pathname, strerror (errno));
 279         exit (1);
 280     }
 281 }
 282
 283 char *
 284 cairo_test_reference_image_filename (const cairo_test_context_t *ctx,
 285                                      const char *base_name,
 286                                      const char *test_name,
 287                                      const char *target_name,
 288                                      const char *format)
 289 {
 290     char *ref_name = NULL;
 291
 292     /* First look for a previous build for comparison. */
 293     if (ctx->refdir != NULL) {
 294         xasprintf (&ref_name, "%s/%s%s",
 295                    ctx->refdir,
 296                    base_name,
 297                    CAIRO_TEST_PNG_SUFFIX);
 298         if (access (ref_name, F_OK) != 0)
 299             free (ref_name);
 300         else
 301             goto done;
 302     }
 303
 304     /* Next look for a target/format-specific reference image. */
 305     xasprintf (&ref_name, "%s/%s-%s-%s%s", ctx->srcdir,
 306                test_name,
 307                target_name,
 308                format,
 309                CAIRO_TEST_REF_SUFFIX);
 310     if (access (ref_name, F_OK) != 0)
 311         free (ref_name);
 312     else
 313         goto done;
 314
 315     /* Next, look for target-specific reference image. */
 316     xasprintf (&ref_name, "%s/%s-%s%s", ctx->srcdir,
 317                test_name,
 318                target_name,
 319                CAIRO_TEST_REF_SUFFIX);
 320     if (access (ref_name, F_OK) != 0)
 321         free (ref_name);
 322     else
 323         goto done;
 324
 325     /* Next, look for format-specific reference image. */
 326     xasprintf (&ref_name, "%s/%s-%s%s", ctx->srcdir,
 327                test_name,
 328                format,
 329                CAIRO_TEST_REF_SUFFIX);
 330     if (access (ref_name, F_OK) != 0)
 331         free (ref_name);
 332     else
 333         goto done;
 334
 335     /* Finally, look for the standard reference image. */
 336     xasprintf (&ref_name, "%s/%s%s", ctx->srcdir,
 337                test_name,
 338                CAIRO_TEST_REF_SUFFIX);
 339     if (access (ref_name, F_OK) != 0)
 340         free (ref_name);
 341     else
 342         goto done;
 343
 344     ref_name = NULL;
 345
 346 done:
 347     return ref_name;
 348 }
 349
 350 static cairo_bool_t
 351 cairo_test_target_has_similar (const cairo_test_context_t *ctx,
 352                                const cairo_boilerplate_target_t *target)
 353 {
 354     cairo_surface_t *surface;
 355     cairo_bool_t has_similar;
 356     cairo_t * cr;
 357     cairo_surface_t *similar;
 358     cairo_status_t status;
 359     void *closure;
 360
 361     /* ignore image intermediate targets */
 362     if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
 363         return FALSE;
 364
 365     if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
 366         return FALSE;
 367
 368     do {
 369         do {
 370             surface = (target->create_surface) (ctx->test->name,
 371                                                 target->content,
 372                                                 ctx->test->width,
 373                                                 ctx->test->height,
 374                                                 ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
 375                                                 ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
 376                                                 CAIRO_BOILERPLATE_MODE_TEST,
 377                                                 0,
 378                                                 &closure);
 379             if (surface == NULL)
 380                 return FALSE;
 381         } while (cairo_test_malloc_failure (ctx, cairo_surface_status (surface)));
 382
 383         if (cairo_surface_status (surface))
 384             return FALSE;
 385
 386
 387         cr = cairo_create (surface);
 388         cairo_push_group_with_content (cr,
 389                                        cairo_boilerplate_content (target->content));
 390         similar = cairo_get_group_target (cr);
 391         status = cairo_surface_status (similar);
 392
 393         has_similar = cairo_surface_get_type (similar) == cairo_surface_get_type (surface);
 394
 395         cairo_destroy (cr);
 396         cairo_surface_destroy (surface);
 397
 398         if (target->cleanup)
 399             target->cleanup (closure);
 400     } while (cairo_test_malloc_failure (ctx, status));
 401
 402     return has_similar;
 403 }
 404
 405 static cairo_surface_t *
 406 _cairo_test_flatten_reference_image (cairo_test_context_t *ctx,
 407                                      cairo_bool_t flatten)
 408 {
 409     cairo_surface_t *surface;
 410     cairo_t *cr;
 411
 412     if (! flatten)
 413         return ctx->ref_image;
 414
 415     if (ctx->ref_image_flattened != NULL)
 416         return ctx->ref_image_flattened;
 417
 418     surface = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
 419                                           cairo_image_surface_get_width (ctx->ref_image),
 420                                           cairo_image_surface_get_height (ctx->ref_image));
 421     cr = cairo_create (surface);
 422     cairo_surface_destroy (surface);
 423
 424     cairo_set_source_rgb (cr, 1, 1, 1);
 425     cairo_paint (cr);
 426
 427     cairo_set_source_surface (cr, ctx->ref_image, 0, 0);
 428     cairo_paint (cr);
 429
 430     surface = cairo_surface_reference (cairo_get_target (cr));
 431     cairo_destroy (cr);
 432
 433     if (cairo_surface_status (surface) == CAIRO_STATUS_SUCCESS)
 434         ctx->ref_image_flattened = surface;
 435     return surface;
 436 }
 437
 438 cairo_surface_t *
 439 cairo_test_get_reference_image (cairo_test_context_t *ctx,
 440                                 const char *filename,
 441                                 cairo_bool_t flatten)
 442 {
 443     cairo_surface_t *surface;
 444     int len;
 445
 446     if (ctx->ref_name != NULL) {
 447         if (strcmp (ctx->ref_name, filename) == 0)
 448             return _cairo_test_flatten_reference_image (ctx, flatten);
 449
 450         cairo_surface_destroy (ctx->ref_image);
 451         ctx->ref_image = NULL;
 452
 453         cairo_surface_destroy (ctx->ref_image_flattened);
 454         ctx->ref_image_flattened = NULL;
 455
 456         free (ctx->ref_name);
 457         ctx->ref_name = NULL;
 458     }
 459
 460     surface = cairo_image_surface_create_from_png (filename);
 461     if (cairo_surface_status (surface))
 462         return surface;
 463
 464     len = strlen (filename);
 465     ctx->ref_name = xmalloc (len + 1);
 466     memcpy (ctx->ref_name, filename, len + 1);
 467
 468     ctx->ref_image = surface;
 469     return _cairo_test_flatten_reference_image (ctx, flatten);
 470 }
 471
 472 static cairo_bool_t
 473 cairo_test_file_is_older (const char *filename,
 474                           const char *ref_filename)
 475 {
 476 #if HAVE_SYS_STAT_H
 477     struct stat st, ref;
 478
 479     if (stat (filename, &st) < 0)
 480         return FALSE;
 481
 482     if (stat (ref_filename, &ref) < 0)
 483         return TRUE;
 484
 485     return st.st_mtime < ref.st_mtime;
 486 #else
 487     /* XXX */
 488     return FALSE;
 489 #endif
 490 }
 491
 492 static cairo_bool_t
 493 cairo_test_files_equal (const char *test_filename,
 494                         const char *pass_filename)
 495 {
 496     FILE *test, *pass;
 497     int t, p;
 498
 499     test = fopen (test_filename, "rb");
 500     if (test == NULL)
 501         return FALSE;
 502
 503     pass = fopen (pass_filename, "rb");
 504     if (pass == NULL) {
 505         fclose (test);
 506         return FALSE;
 507     }
 508
 509     /* as simple as it gets */
 510     do {
 511         t = getc (test);
 512         p = getc (pass);
 513         if (t != p)
 514             break;
 515     } while (t != EOF && p != EOF);
 516
 517     fclose (pass);
 518     fclose (test);
 519
 520     return t == p; /* both EOF */
 521 }
 522
 523 static cairo_bool_t
 524 cairo_test_copy_file (const char *src_filename,
 525                       const char *dst_filename)
 526 {
 527     FILE *src, *dst;
 528     int c;
 529
 530 #if HAVE_LINK
 531     if (link (src_filename, dst_filename) == 0)
 532         return TRUE;
 533
 534     unlink (dst_filename);
 535 #endif
 536
 537     src = fopen (src_filename, "rb");
 538     if (src == NULL)
 539         return FALSE;
 540
 541     dst = fopen (dst_filename, "wb");
 542     if (dst == NULL) {
 543         fclose (src);
 544         return FALSE;
 545     }
 546
 547     /* as simple as it gets */
 548     while ((c = getc (src)) != EOF)
 549         putc (c, dst);
 550
 551     fclose (src);
 552     fclose (dst);
 553
 554     return TRUE;
 555 }
 556
 557 static cairo_test_status_t
 558 cairo_test_for_target (cairo_test_context_t              *ctx,
 559                        const cairo_boilerplate_target_t  *target,
 560                        int                                dev_offset,
 561                        cairo_bool_t                       similar)
 562 {
 563     cairo_test_status_t status;
 564     cairo_surface_t *surface = NULL;
 565     cairo_t *cr;
 566     const char *empty_str = "";
 567     char *offset_str, *thread_str;
 568     char *base_name, *png_name, *ref_name, *diff_name;
 569     char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
 570     cairo_test_status_t ret;
 571     cairo_content_t expected_content;
 572     cairo_font_options_t *font_options;
 573     const char *format;
 574     cairo_bool_t have_output = FALSE;
 575     cairo_bool_t have_result = FALSE;
 576     int malloc_failure_iterations = ctx->malloc_failure;
 577     void *closure;
 578     int width, height;
 579     int last_fault_count = 0;
 580
 581     /* Get the strings ready that we'll need. */
 582     format = cairo_boilerplate_content_name (target->content);
 583     if (dev_offset)
 584         xasprintf (&offset_str, "-%d", dev_offset);
 585     else
 586         offset_str = (char *) empty_str;
 587     if (ctx->thread)
 588         xasprintf (&thread_str, "-thread%d", ctx->thread);
 589     else
 590         thread_str = (char *) empty_str;
 591
 592     xasprintf (&base_name, "%s-%s-%s%s%s%s",
 593                ctx->test->name,
 594                target->name,
 595                format,
 596                similar ? "-similar" : "",
 597                offset_str,
 598                thread_str);
 599
 600     if (offset_str != empty_str)
 601       free (offset_str);
 602     if (thread_str != empty_str)
 603       free (thread_str);
 604
 605
 606     ref_name = cairo_test_reference_image_filename (ctx,
 607                                                     base_name,
 608                                                     ctx->test->name,
 609                                                     target->name,
 610                                                     format);
 611     xasprintf (&png_name,  "%s%s", base_name, CAIRO_TEST_PNG_SUFFIX);
 612     xasprintf (&diff_name, "%s%s", base_name, CAIRO_TEST_DIFF_SUFFIX);
 613
 614     if (target->is_vector) {
 615         int i;
 616
 617         for (i = 0; vector_ignored_tests[i] != NULL; i++)
 618             if (strcmp (ctx->test->name, vector_ignored_tests[i]) == 0) {
 619                 cairo_test_log (ctx, "Error: Skipping for vector target %s\n", target->name);
 620                 ret = CAIRO_TEST_UNTESTED;
 621                 goto UNWIND_STRINGS;
 622             }
 623     }
 624
 625     width = ctx->test->width;
 626     height = ctx->test->height;
 627     if (width && height) {
 628         width += dev_offset;
 629         height += dev_offset;
 630     }
 631
 632 REPEAT:
 633 #if HAVE_MEMFAULT
 634     VALGRIND_CLEAR_FAULTS ();
 635     VALGRIND_RESET_LEAKS ();
 636     ctx->last_fault_count = 0;
 637     last_fault_count = VALGRIND_COUNT_FAULTS ();
 638     VALGRIND_ENABLE_FAULTS ();
 639 #endif
 640     have_output = FALSE;
 641     have_result = FALSE;
 642
 643     /* Run the actual drawing code. */
 644     ret = CAIRO_TEST_SUCCESS;
 645     surface = (target->create_surface) (base_name,
 646                                         target->content,
 647                                         width, height,
 648                                         ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
 649                                         ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
 650                                         CAIRO_BOILERPLATE_MODE_TEST,
 651                                         ctx->thread,
 652                                         &closure);
 653     if (surface == NULL) {
 654         cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
 655         ret = CAIRO_TEST_UNTESTED;
 656         goto UNWIND_STRINGS;
 657     }
 658
 659     if (cairo_test_malloc_failure (ctx, cairo_surface_status (surface)))
 660         goto REPEAT;
 661
 662     if (cairo_surface_status (surface)) {
 663         MF (VALGRIND_PRINT_FAULTS ());
 664         cairo_test_log (ctx, "Error: Created an error surface\n");
 665         ret = CAIRO_TEST_FAILURE;
 666         goto UNWIND_STRINGS;
 667     }
 668
 669     /* Check that we created a surface of the expected type. */
 670     if (cairo_surface_get_type (surface) != target->expected_type) {
 671         MF (VALGRIND_PRINT_FAULTS ());
 672         cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
 673                         cairo_surface_get_type (surface), target->expected_type);
 674         ret = CAIRO_TEST_FAILURE;
 675         goto UNWIND_SURFACE;
 676     }
 677
 678     /* Check that we created a surface of the expected content,
 679      * (ignore the artificial CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
 680      */
 681     expected_content = cairo_boilerplate_content (target->content);
 682
 683     if (cairo_surface_get_content (surface) != expected_content) {
 684         MF (VALGRIND_PRINT_FAULTS ());
 685         cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
 686                         cairo_surface_get_content (surface), expected_content);
 687         ret = CAIRO_TEST_FAILURE;
 688         goto UNWIND_SURFACE;
 689     }
 690
 691     cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
 692
 693     cr = cairo_create (surface);
 694     if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
 695 #if HAVE_MEMFAULT
 696         cairo_destroy (cr);
 697         cairo_surface_destroy (surface);
 698
 699         if (target->cleanup)
 700             target->cleanup (closure);
 701
 702         goto REPEAT;
 703 #else
 704         ret = CAIRO_TEST_FAILURE;
 705         goto UNWIND_CAIRO;
 706 #endif
 707     }
 708
 709     if (similar)
 710         cairo_push_group_with_content (cr, expected_content);
 711
 712     /* Clear to transparent (or black) depending on whether the target
 713      * surface supports alpha. */
 714     cairo_save (cr);
 715     cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
 716     cairo_paint (cr);
 717     cairo_restore (cr);
 718
 719     /* Set all components of font_options to avoid backend differences
 720      * and reduce number of needed reference images. */
 721     font_options = cairo_font_options_create ();
 722     cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
 723     cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
 724     cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
 725     cairo_set_font_options (cr, font_options);
 726     cairo_font_options_destroy (font_options);
 727
 728     cairo_save (cr);
 729     status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
 730     cairo_restore (cr);
 731
 732     if (similar) {
 733         cairo_pop_group_to_source (cr);
 734         cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
 735         cairo_paint (cr);
 736     }
 737
 738 #if HAVE_MEMFAULT
 739     VALGRIND_DISABLE_FAULTS ();
 740
 741     /* repeat test after malloc failure injection */
 742     if (ctx->malloc_failure &&
 743         VALGRIND_COUNT_FAULTS () - last_fault_count > 0 &&
 744         (status == CAIRO_TEST_NO_MEMORY ||
 745          cairo_status (cr) == CAIRO_STATUS_NO_MEMORY ||
 746          cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY))
 747     {
 748         cairo_destroy (cr);
 749         cairo_surface_destroy (surface);
 750         if (target->cleanup)
 751             target->cleanup (closure);
 752         if (ctx->thread == 0) {
 753             cairo_debug_reset_static_data ();
 754 #if HAVE_FCFINI
 755             FcFini ();
 756 #endif
 757             if (VALGRIND_COUNT_LEAKS () > 0) {
 758                 VALGRIND_PRINT_FAULTS ();
 759                 VALGRIND_PRINT_LEAKS ();
 760             }
 761         }
 762
 763         goto REPEAT;
 764     }
 765 #endif
 766
 767     /* Then, check all the different ways it could fail. */
 768     if (status) {
 769         cairo_test_log (ctx, "Error: Function under test failed\n");
 770         ret = status;
 771         goto UNWIND_CAIRO;
 772     }
 773
 774     if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
 775         cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
 776                         cairo_status_to_string (cairo_status (cr)));
 777         ret = CAIRO_TEST_FAILURE;
 778         goto UNWIND_CAIRO;
 779     }
 780
 781 #if HAVE_MEMFAULT
 782     if (VALGRIND_COUNT_FAULTS () - last_fault_count > 0) {
 783         VALGRIND_PRINTF ("Unreported memfaults...");
 784         VALGRIND_PRINT_FAULTS ();
 785     }
 786 #endif
 787
 788     /* Skip image check for tests with no image (width,height == 0,0) */
 789     if (ctx->test->width != 0 && ctx->test->height != 0) {
 790         cairo_surface_t *ref_image;
 791         cairo_surface_t *test_image;
 792         cairo_surface_t *diff_image;
 793         buffer_diff_result_t result;
 794         cairo_status_t diff_status;
 795
 796         if (target->finish_surface != NULL) {
 797             diff_status = target->finish_surface (surface);
 798             if (diff_status) {
 799                 cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
 800                                 cairo_status_to_string (diff_status));
 801                 ret = CAIRO_TEST_FAILURE;
 802                 goto UNWIND_CAIRO;
 803             }
 804         }
 805
 806         if (ref_name == NULL) {
 807             cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
 808                             base_name);
 809
 810             /* we may be running this test to generate reference images */
 811             _xunlink (ctx, png_name);
 812             test_image = target->get_image_surface (surface, 0,
 813                                                     ctx->test->width,
 814                                                     ctx->test->height);
 815             diff_status = cairo_surface_write_to_png (test_image, png_name);
 816             if (diff_status) {
 817                 cairo_test_log (ctx,
 818                                 "Error: Failed to write output image: %s\n",
 819                                 cairo_status_to_string (diff_status));
 820             }
 821             have_output = TRUE;
 822             cairo_surface_destroy (test_image);
 823
 824             ret = CAIRO_TEST_FAILURE;
 825             goto UNWIND_CAIRO;
 826         }
 827
 828         if (target->file_extension != NULL) { /* compare vector surfaces */
 829             xasprintf (&test_filename, "%s-out%s",
 830                        base_name, target->file_extension);
 831             xasprintf (&pass_filename, "%s-pass%s",
 832                        base_name, target->file_extension);
 833             xasprintf (&fail_filename, "%s-fail%s",
 834                        base_name, target->file_extension);
 835
 836             if (cairo_test_file_is_older (pass_filename, ref_name))
 837                 _xunlink (ctx, pass_filename);
 838             if (cairo_test_file_is_older (fail_filename, ref_name))
 839                 _xunlink (ctx, fail_filename);
 840
 841             if (cairo_test_files_equal (test_filename, pass_filename)) {
 842                 /* identical output as last known PASS */
 843                 cairo_test_log (ctx, "Vector surface matches last pass.\n");
 844                 have_output = TRUE;
 845                 ret = CAIRO_TEST_SUCCESS;
 846                 goto UNWIND_CAIRO;
 847             }
 848             if (cairo_test_files_equal (test_filename, fail_filename)) {
 849                 /* identical output as last known FAIL, fail */
 850                 cairo_test_log (ctx, "Vector surface matches last fail.\n");
 851                 have_result = TRUE; /* presume these were kept around as well */
 852                 have_output = TRUE;
 853                 ret = CAIRO_TEST_FAILURE;
 854                 goto UNWIND_CAIRO;
 855             }
 856         }
 857
 858         test_image = target->get_image_surface (surface, 0,
 859                                                 ctx->test->width,
 860                                                 ctx->test->height);
 861         if (cairo_surface_status (test_image)) {
 862             cairo_test_log (ctx, "Error: Failed to extract image: %s\n",
 863                             cairo_status_to_string (cairo_surface_status (test_image)));
 864             cairo_surface_destroy (test_image);
 865             ret = CAIRO_TEST_FAILURE;
 866             goto UNWIND_CAIRO;
 867         }
 868
 869         _xunlink (ctx, png_name);
 870         diff_status = cairo_surface_write_to_png (test_image, png_name);
 871         if (diff_status) {
 872             cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
 873                             cairo_status_to_string (diff_status));
 874             cairo_surface_destroy (test_image);
 875             ret = CAIRO_TEST_FAILURE;
 876             goto UNWIND_CAIRO;
 877         }
 878         have_output = TRUE;
 879
 880         /* binary compare png files (no decompression) */
 881         if (target->file_extension == NULL) {
 882             xasprintf (&test_filename, "%s", png_name);
 883             xasprintf (&pass_filename, "%s-pass.png", base_name);
 884             xasprintf (&fail_filename, "%s-fail.png", base_name);
 885
 886             if (cairo_test_file_is_older (pass_filename, ref_name))
 887                 _xunlink (ctx, pass_filename);
 888             if (cairo_test_file_is_older (fail_filename, ref_name))
 889                 _xunlink (ctx, fail_filename);
 890
 891             if (cairo_test_files_equal (test_filename, pass_filename)) {
 892                 /* identical output as last known PASS, pass */
 893                 cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
 894                 cairo_surface_destroy (test_image);
 895                 ret = CAIRO_TEST_SUCCESS;
 896                 goto UNWIND_CAIRO;
 897             }
 898             if (cairo_test_files_equal (png_name, ref_name)) {
 899                 /* identical output as reference image */
 900                 cairo_test_log (ctx, "PNG file exactly reference image.\n");
 901                 cairo_surface_destroy (test_image);
 902                 ret = CAIRO_TEST_SUCCESS;
 903                 goto UNWIND_CAIRO;
 904             }
 905
 906             if (cairo_test_files_equal (test_filename, fail_filename)) {
 907                 cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
 908                 /* identical output as last known FAIL, fail */
 909                 have_result = TRUE; /* presume these were kept around as well */
 910                 cairo_surface_destroy (test_image);
 911                 ret = CAIRO_TEST_FAILURE;
 912                 goto UNWIND_CAIRO;
 913             }
 914         } else {
 915             if (cairo_test_files_equal (png_name, ref_name)) {
 916                 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
 917                 cairo_surface_destroy (test_image);
 918                 ret = CAIRO_TEST_SUCCESS;
 919                 goto UNWIND_CAIRO;
 920             }
 921         }
 922
 923         ref_image = cairo_test_get_reference_image (ctx, ref_name,
 924                                                     target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
 925         if (cairo_surface_status (ref_image)) {
 926             cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
 927                             ref_name,
 928                             cairo_status_to_string (cairo_surface_status (ref_image)));
 929             cairo_surface_destroy (ref_image);
 930             cairo_surface_destroy (test_image);
 931             ret = CAIRO_TEST_FAILURE;
 932             goto UNWIND_CAIRO;
 933         }
 934
 935         diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
 936                                                  ctx->test->width,
 937                                                  ctx->test->height);
 938
 939         diff_status = image_diff (ctx,
 940                                   test_image, ref_image,
 941                                   diff_image,
 942                                   &result);
 943         _xunlink (ctx, diff_name);
 944         if (diff_status) {
 945             cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
 946                             cairo_status_to_string (diff_status));
 947             ret = CAIRO_TEST_FAILURE;
 948         }
 949         else if (result.pixels_changed &&
 950                  result.max_diff > target->error_tolerance)
 951         {
 952             ret = CAIRO_TEST_FAILURE;
 953
 954             diff_status = cairo_surface_write_to_png (diff_image, diff_name);
 955             if (diff_status) {
 956                 cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
 957                                 cairo_status_to_string (diff_status));
 958             } else
 959                 have_result = TRUE;
 960
 961             cairo_test_copy_file (test_filename, fail_filename);
 962         } else { /* success */
 963             cairo_test_copy_file (test_filename, pass_filename);
 964         }
 965
 966         cairo_surface_destroy (test_image);
 967         cairo_surface_destroy (diff_image);
 968     }
 969
 970 UNWIND_CAIRO:
 971     if (test_filename != NULL) {
 972         free (test_filename);
 973         test_filename = NULL;
 974     }
 975     if (fail_filename != NULL) {
 976         free (fail_filename);
 977         fail_filename = NULL;
 978     }
 979     if (pass_filename != NULL) {
 980         free (pass_filename);
 981         pass_filename = NULL;
 982     }
 983
 984 #if HAVE_MEMFAULT
 985     if (ret == CAIRO_TEST_FAILURE && ctx->expectation != CAIRO_TEST_FAILURE)
 986         VALGRIND_PRINT_FAULTS ();
 987 #endif
 988     cairo_destroy (cr);
 989 UNWIND_SURFACE:
 990     cairo_surface_destroy (surface);
 991
 992     if (target->cleanup)
 993         target->cleanup (closure);
 994
 995 #if HAVE_MEMFAULT
 996     if (ctx->thread == 0) {
 997         cairo_debug_reset_static_data ();
 998
 999 #if HAVE_FCFINI
1000         FcFini ();
1001 #endif
1002
1003         if (VALGRIND_COUNT_LEAKS () > 0) {
1004             if (ret != CAIRO_TEST_FAILURE ||
1005                 ctx->expectation == CAIRO_TEST_FAILURE)
1006             {
1007                 VALGRIND_PRINT_FAULTS ();
1008             }
1009             VALGRIND_PRINT_LEAKS ();
1010         }
1011     }
1012
1013     if (ret == CAIRO_TEST_SUCCESS && --malloc_failure_iterations > 0)
1014         goto REPEAT;
1015 #endif
1016
1017     if (ctx->thread == 0) {
1018         if (have_output)
1019             cairo_test_log (ctx, "OUTPUT: %s\n", png_name);
1020
1021         if (have_result) {
1022             cairo_test_log (ctx,
1023                             "REFERENCE: %s\nDIFFERENCE: %s\n",
1024                             ref_name, diff_name);
1025         }
1026     }
1027
1028 UNWIND_STRINGS:
1029     if (png_name)
1030       free (png_name);
1031     if (ref_name)
1032       free (ref_name);
1033     if (diff_name)
1034       free (diff_name);
1035     if (base_name)
1036       free (base_name);
1037
1038     return ret;
1039 }
1040
1041 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1042 #include <signal.h>
1043 #include <setjmp.h>
1044 /* Used to catch crashes in a test, so that we report it as such and
1045  * continue testing, although one crasher may already have corrupted memory in
1046  * an nonrecoverable fashion. */
1047 static jmp_buf jmpbuf;
1048
1049 static void
1050 segfault_handler (int signal)
1051 {
1052     longjmp (jmpbuf, signal);
1053 }
1054 #endif
1055
1056 static cairo_test_status_t
1057 cairo_test_run (cairo_test_context_t *ctx)
1058 {
1059     /* we use volatile here to make sure values are not clobbered
1060      * by longjmp */
1061     volatile size_t i, j;
1062     volatile cairo_bool_t print_fail_on_stdout = ctx->thread == 0;
1063     volatile cairo_test_status_t status, ret;
1064
1065 #if HAVE_UNISTD_H
1066     if (ctx->thread == 0 && isatty (2)) {
1067         fail_face = "\033[41m\033[37m\033[1m";
1068         normal_face = "\033[m";
1069         if (isatty (1))
1070             print_fail_on_stdout = FALSE;
1071     }
1072 #endif
1073
1074     /* The intended logic here is that we return overall SUCCESS
1075      * iff. there is at least one tested backend and that all tested
1076      * backends return SUCCESS, OR, there's backends were manually
1077      * limited, and none were tested.
1078      * In other words:
1079      *
1080      *  if      backends limited and no backend tested
1081      *          -> SUCCESS
1082      *  else if any backend not SUCCESS
1083      *          -> FAILURE
1084      *  else if all backends UNTESTED
1085      *          -> FAILURE
1086      *  else    (== some backend SUCCESS)
1087      *          -> SUCCESS
1088      *
1089      * Also, on a crash, run no further tests.
1090      */
1091     status = ret = CAIRO_TEST_UNTESTED;
1092     for (i = 0; i < ctx->num_targets && status != CAIRO_TEST_CRASHED; i++) {
1093         const cairo_boilerplate_target_t * volatile target = ctx->targets_to_test[(i + ctx->thread) % ctx->num_targets];
1094
1095         for (j = 0; j < NUM_DEVICE_OFFSETS; j++) {
1096             volatile int dev_offset = ((j + ctx->thread) % NUM_DEVICE_OFFSETS) * 25;
1097             volatile int similar, has_similar;
1098
1099             has_similar = cairo_test_target_has_similar (ctx, target);
1100             for (similar = 0; similar <= has_similar ; similar++) {
1101                 cairo_test_log (ctx, "Testing %s with %s%s target (dev offset %d)\n", ctx->test_name, similar ? " (similar) " : "", target->name, dev_offset);
1102                 if (ctx->thread == 0) {
1103                     printf ("%s-%s-%s [%d]%s:\t", ctx->test->name, target->name,
1104                             cairo_boilerplate_content_name (target->content),
1105                             dev_offset,
1106                             similar ? " (similar) ": "");
1107                     fflush (stdout);
1108                 }
1109
1110 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1111                 if (ctx->thread == 0 && ! RUNNING_ON_VALGRIND) {
1112                     void (* volatile old_segfault_handler)(int);
1113                     void (* volatile old_sigpipe_handler)(int);
1114
1115                     /* Set up a checkpoint to get back to in case of segfaults. */
1116 #ifdef SIGSEGV
1117                     old_segfault_handler = signal (SIGSEGV, segfault_handler);
1118 #endif
1119 #ifdef SIGPIPE
1120                     old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
1121 #endif
1122                     if (0 == setjmp (jmpbuf))
1123                         status = cairo_test_for_target (ctx, target, dev_offset, similar);
1124                     else
1125                         status = CAIRO_TEST_CRASHED;
1126 #ifdef SIGSEGV
1127                     signal (SIGSEGV, old_segfault_handler);
1128 #endif
1129 #ifdef SIGPIPE
1130                     signal (SIGPIPE, old_sigpipe_handler);
1131 #endif
1132                 } else {
1133                     status = cairo_test_for_target (ctx, target, dev_offset, similar);
1134                 }
1135 #else
1136                 status = cairo_test_for_target (ctx, target, dev_offset, similar);
1137 #endif
1138
1139                 if (ctx->thread == 0) {
1140                     cairo_test_log (ctx,
1141                             "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
1142                                     ctx->test->name, target->name,
1143                                     cairo_boilerplate_content_name (target->content),
1144                                     dev_offset, similar);
1145                     switch (status) {
1146                     case CAIRO_TEST_SUCCESS:
1147                         printf ("PASS\n");
1148                         cairo_test_log (ctx, "PASS\n");
1149                         if (ret == CAIRO_TEST_UNTESTED)
1150                             ret = CAIRO_TEST_SUCCESS;
1151                         break;
1152                     case CAIRO_TEST_UNTESTED:
1153                         printf ("UNTESTED\n");
1154                         cairo_test_log (ctx, "UNTESTED\n");
1155                         break;
1156                     case CAIRO_TEST_CRASHED:
1157                         if (print_fail_on_stdout) {
1158                             printf ("!!!CRASHED!!!\n");
1159                         } else {
1160                         /* eat the test name */
1161                         printf ("\r");
1162                         fflush (stdout);
1163                         }
1164                         cairo_test_log (ctx, "CRASHED\n");
1165                         fprintf (stderr, "%s-%s-%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
1166                                  ctx->test->name, target->name,
1167                                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1168                                  fail_face, normal_face);
1169                         ret = CAIRO_TEST_FAILURE;
1170                         break;
1171                     default:
1172                     case CAIRO_TEST_NO_MEMORY:
1173                     case CAIRO_TEST_FAILURE:
1174                         if (ctx->expectation == CAIRO_TEST_FAILURE) {
1175                             printf ("XFAIL\n");
1176                             cairo_test_log (ctx, "XFAIL\n");
1177                         } else {
1178                             if (print_fail_on_stdout) {
1179                                 printf ("FAIL\n");
1180                             } else {
1181                                 /* eat the test name */
1182                                 printf ("\r");
1183                                 fflush (stdout);
1184                             }
1185                             fprintf (stderr, "%s-%s-%s [%d]%s:\t%sFAIL%s\n",
1186                                      ctx->test->name, target->name,
1187                                      cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1188                                      fail_face, normal_face);
1189                             cairo_test_log (ctx, "FAIL\n");
1190                         }
1191                         ret = CAIRO_TEST_FAILURE;
1192                         break;
1193                     }
1194                     fflush (stdout);
1195                 } else {
1196 #if _POSIX_THREAD_SAFE_FUNCTIONS
1197                     flockfile (stdout);
1198 #endif
1199                     printf ("%s-%s-%s %d [%d]:\t",
1200                             ctx->test->name, target->name,
1201                             cairo_boilerplate_content_name (target->content),
1202                             ctx->thread,
1203                             dev_offset);
1204                     switch (status) {
1205                     case CAIRO_TEST_SUCCESS:
1206                         printf ("PASS\n");
1207                         break;
1208                     case CAIRO_TEST_UNTESTED:
1209                         printf ("UNTESTED\n");
1210                         break;
1211                     case CAIRO_TEST_CRASHED:
1212                         printf ("!!!CRASHED!!!\n");
1213                         ret = CAIRO_TEST_FAILURE;
1214                         break;
1215                     default:
1216                     case CAIRO_TEST_NO_MEMORY:
1217                     case CAIRO_TEST_FAILURE:
1218                         if (ctx->expectation == CAIRO_TEST_FAILURE) {
1219                             printf ("XFAIL\n");
1220                         } else {
1221                             printf ("FAIL\n");
1222                         }
1223                         ret = CAIRO_TEST_FAILURE;
1224                         break;
1225                     }
1226
1227                     fflush (stdout);
1228 #if _POSIX_THREAD_SAFE_FUNCTIONS
1229                     funlockfile (stdout);
1230 #endif
1231                 }
1232             }
1233         }
1234     }
1235
1236     return ret;
1237 }
1238
1239 #if HAVE_PTHREAD_H
1240 typedef struct _cairo_test_thread {
1241     pthread_t thread;
1242     cairo_test_context_t *ctx;
1243     size_t id;
1244 } cairo_test_thread_t;
1245
1246 static void *
1247 cairo_test_run_threaded (void *closure)
1248 {
1249     cairo_test_thread_t *arg = closure;
1250     cairo_test_context_t ctx;
1251     cairo_test_status_t ret;
1252
1253     cairo_test_init_thread (&ctx, arg->ctx, arg->id);
1254
1255     ret = cairo_test_run (&ctx);
1256
1257     cairo_test_fini (&ctx);
1258
1259     return (void *) ret;
1260 }
1261 #endif
1262
1263
1264 static cairo_test_status_t
1265 cairo_test_expecting (const cairo_test_t *test,
1266                       cairo_test_status_t expectation)
1267 {
1268     cairo_test_context_t ctx;
1269     cairo_test_status_t ret = CAIRO_TEST_SUCCESS;
1270     size_t num_threads;
1271
1272     _cairo_test_init (&ctx, test, test->name, expectation);
1273     printf ("%s\n", test->description);
1274
1275     if (expectation == CAIRO_TEST_FAILURE)
1276         printf ("Expecting failure\n");
1277
1278 #if HAVE_PTHREAD_H
1279     num_threads = 0;
1280     if (getenv ("CAIRO_TEST_NUM_THREADS"))
1281         num_threads = atoi (getenv ("CAIRO_TEST_NUM_THREADS"));
1282     if (num_threads > 1) {
1283         cairo_test_thread_t *threads;
1284         size_t n;
1285
1286         threads = xmalloc (sizeof (cairo_test_thread_t) * num_threads);
1287         for (n = 0; n < num_threads; n++) {
1288             threads[n].ctx = &ctx;
1289             threads[n].id = n + 1;
1290             pthread_create (&threads[n].thread, NULL,
1291                     cairo_test_run_threaded, &threads[n]);
1292         }
1293         for (n = 0; n < num_threads; n++) {
1294             void *tmp;
1295             pthread_join (threads[n].thread, &tmp);
1296             if (ret == CAIRO_TEST_SUCCESS)
1297                 ret = (cairo_test_status_t) tmp;
1298         }
1299         free (threads);
1300     }
1301
1302     if (ret == CAIRO_TEST_SUCCESS)
1303 #endif
1304         ret = cairo_test_run (&ctx);
1305
1306     if (ret != CAIRO_TEST_SUCCESS)
1307         printf ("Check %s%s out for more information.\n", test->name, CAIRO_TEST_LOG_SUFFIX);
1308
1309     /* if the set of targets to test was limited using CAIRO_TEST_TARGET, we
1310      * behave slightly differently, to ensure that limiting the targets does
1311      * not increase the number of tests failing. */
1312     if (ctx.limited_targets) {
1313         /* if all passed, but expecting failure, return failure to not
1314          * trigger an XPASS failure */
1315         if (expectation == CAIRO_TEST_FAILURE && ret == CAIRO_TEST_SUCCESS) {
1316             printf ("All tested backends passed, but tested targets are manually limited\n"
1317                     "and the test suite expects this test to fail for at least one target.\n"
1318                     "Intentionally failing the test, to not fail the suite.\n");
1319             ret = CAIRO_TEST_FAILURE;
1320         }
1321     }
1322
1323     cairo_test_fini (&ctx);
1324
1325     return ret;
1326 }
1327
1328 cairo_test_status_t
1329 cairo_test (const cairo_test_t *test)
1330 {
1331     cairo_test_status_t expectation = CAIRO_TEST_SUCCESS;
1332     const char *xfails;
1333
1334 #ifdef _MSC_VER
1335     /* We don't want an assert dialog, we want stderr */
1336     _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
1337     _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
1338 #endif
1339
1340     if ((xfails = getenv ("CAIRO_XFAIL_TESTS")) != NULL) {
1341         while (*xfails) {
1342             const char *end = strpbrk (xfails, " \t\r\n;:,");
1343             if (!end)
1344                 end = xfails + strlen (xfails);
1345
1346             if (0 == strncmp (test->name, xfails, end - xfails) &&
1347                 '\0' == test->name[end - xfails]) {
1348                 expectation = CAIRO_TEST_FAILURE;
1349                 break;
1350             }
1351
1352             if (*end)
1353               end++;
1354             xfails = end;
1355         }
1356     }
1357
1358     return cairo_test_expecting (test, expectation);
1359 }
1360
1361 const cairo_test_context_t *
1362 cairo_test_get_context (cairo_t *cr)
1363 {
1364     return cairo_get_user_data (cr, &_cairo_test_context_key);
1365 }
1366
1367 cairo_surface_t *
1368 cairo_test_create_surface_from_png (const cairo_test_context_t *ctx,
1369                                     const char *filename)
1370 {
1371     cairo_surface_t *image;
1372     cairo_status_t status;
1373
1374     image = cairo_image_surface_create_from_png (filename);
1375     status = cairo_surface_status (image);
1376     if (status == CAIRO_STATUS_FILE_NOT_FOUND) {
1377         /* expect not found when running with srcdir != builddir
1378          * such as when 'make distcheck' is run
1379          */
1380         if (ctx->srcdir) {
1381             char *srcdir_filename;
1382             xasprintf (&srcdir_filename, "%s/%s", ctx->srcdir, filename);
1383             image = cairo_image_surface_create_from_png (srcdir_filename);
1384             free (srcdir_filename);
1385         }
1386     }
1387
1388     return image;
1389 }
1390
1391 cairo_pattern_t *
1392 cairo_test_create_pattern_from_png (const cairo_test_context_t *ctx,
1393                                     const char *filename)
1394 {
1395     cairo_surface_t *image;
1396     cairo_pattern_t *pattern;
1397
1398     image = cairo_test_create_surface_from_png (ctx, filename);
1399
1400     pattern = cairo_pattern_create_for_surface (image);
1401
1402     cairo_pattern_set_extend (pattern, CAIRO_EXTEND_REPEAT);
1403
1404     cairo_surface_destroy (image);
1405
1406     return pattern;
1407 }
1408
1409 static cairo_surface_t *
1410 _draw_check (int width, int height)
1411 {
1412     cairo_surface_t *surface;
1413     cairo_t *cr;
1414
1415     surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, 12, 12);
1416     cr = cairo_create (surface);
1417     cairo_surface_destroy (surface);
1418
1419     cairo_set_source_rgb (cr, 0.75, 0.75, 0.75); /* light gray */
1420     cairo_paint (cr);
1421
1422     cairo_set_source_rgb (cr, 0.25, 0.25, 0.25); /* dark gray */
1423     cairo_rectangle (cr, width / 2,  0, width / 2, height / 2);
1424     cairo_rectangle (cr, 0, height / 2, width / 2, height / 2);
1425     cairo_fill (cr);
1426
1427     surface = cairo_surface_reference (cairo_get_target (cr));
1428     cairo_destroy (cr);
1429
1430     return surface;
1431 }
1432
1433 void
1434 cairo_test_paint_checkered (cairo_t *cr)
1435 {
1436     cairo_surface_t *check;
1437
1438     check = _draw_check (12, 12);
1439
1440     cairo_save (cr);
1441     cairo_set_source_surface (cr, check, 0, 0);
1442     cairo_surface_destroy (check);
1443
1444     cairo_pattern_set_filter (cairo_get_source (cr), CAIRO_FILTER_NEAREST);
1445     cairo_pattern_set_extend (cairo_get_source (cr), CAIRO_EXTEND_REPEAT);
1446     cairo_paint (cr);
1447
1448     cairo_restore (cr);
1449 }
1450
1451 cairo_bool_t
1452 cairo_test_is_target_enabled (const cairo_test_context_t *ctx, const char *target)
1453 {
1454     size_t i;
1455
1456     for (i = 0; i < ctx->num_targets; i++) {
1457         const cairo_boilerplate_target_t *t = ctx->targets_to_test[i];
1458         if (strcmp (t->name, target) == 0) {
1459             /* XXX ask the target whether is it possible to run?
1460              * e.g. the xlib backend could check whether it is able to connect
1461              * to the Display.
1462              */
1463             return TRUE;
1464         }
1465     }
1466
1467     return FALSE;
1468 }
1469
1470 cairo_bool_t
1471 cairo_test_malloc_failure (const cairo_test_context_t *ctx,
1472                            cairo_status_t status)
1473 {
1474     int n_faults;
1475
1476     if (! ctx->malloc_failure)
1477         return FALSE;
1478
1479     if (status != CAIRO_STATUS_NO_MEMORY)
1480         return FALSE;
1481
1482 #if HAVE_MEMFAULT
1483     /* prevent infinite loops... */
1484     n_faults = VALGRIND_COUNT_FAULTS ();
1485     if (n_faults == ctx->last_fault_count)
1486         return FALSE;
1487
1488     ((cairo_test_context_t *) ctx)->last_fault_count = n_faults;
1489 #endif
1490
1491     return TRUE;
1492 }
1493
1494 cairo_test_status_t
1495 cairo_test_status_from_status (const cairo_test_context_t *ctx,
1496                                cairo_status_t status)
1497 {
1498     if (status == CAIRO_STATUS_SUCCESS)
1499         return CAIRO_TEST_SUCCESS;
1500
1501     if (cairo_test_malloc_failure (ctx, status))
1502         return CAIRO_TEST_NO_MEMORY;
1503
1504     return CAIRO_TEST_FAILURE;
1505 }