cairo/perf/cairo-perf-diff-files.c

   1 /*
   2  * Copyright © 2006 Red Hat, Inc.
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software
   5  * and its documentation for any purpose is hereby granted without
   6  * fee, provided that the above copyright notice appear in all copies
   7  * and that both that copyright notice and this permission notice
   8  * appear in supporting documentation, and that the name of the
   9  * copyright holders not be used in advertising or publicity
  10  * pertaining to distribution of the software without specific,
  11  * written prior permission. The copyright holders make no
  12  * representations about the suitability of this software for any
  13  * purpose.  It is provided "as is" without express or implied
  14  * warranty.
  15  *
  16  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
  17  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  18  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
  19  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  20  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  21  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
  22  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  23  * SOFTWARE.
  24  *
  25  * Authors: Carl Worth <cworth@cworth.org>
  26  */
  27
  28 #include "cairo-perf.h"
  29
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <errno.h>
  34 #include <ctype.h>
  35 #include <math.h>
  36 #include <assert.h>
  37
  38 typedef struct _cairo_perf_report_options {
  39     double min_change;
  40     int use_utf;
  41     int print_change_bars;
  42     int use_ms;
  43 } cairo_perf_report_options_t;
  44
  45 typedef struct _cairo_perf_diff_files_args {
  46     const char **filenames;
  47     int num_filenames;
  48     cairo_perf_report_options_t options;
  49 } cairo_perf_diff_files_args_t;
  50
  51 static int
  52 test_diff_cmp_speedup_before_slowdown (const void *a, const void *b)
  53 {
  54     const test_diff_t *a_diff = a;
  55     const test_diff_t *b_diff = b;
  56
  57     /* First make all speedups come before all slowdowns. */
  58     if (a_diff->change > 1.0 && b_diff->change < 1.0)
  59         return -1;
  60     if (a_diff->change < 1.0 && b_diff->change > 1.0)
  61         return 1;
  62
  63     /* Reverse sort by magnitude of change so larger changes come
  64      * first */
  65     if (fabs (a_diff->change) > fabs (b_diff->change))
  66         return -1;
  67
  68     if (fabs (a_diff->change) < fabs (b_diff->change))
  69         return 1;
  70
  71     return 0;
  72 }
  73
  74 static int
  75 test_diff_cmp (const void *a, const void *b)
  76 {
  77     const test_diff_t *a_diff = a;
  78     const test_diff_t *b_diff = b;
  79
  80     /* Reverse sort by magnitude of change so larger changes come
  81      * first */
  82     if (a_diff->change > b_diff->change)
  83         return -1;
  84
  85     if (a_diff->change < b_diff->change)
  86         return 1;
  87
  88     return 0;
  89 }
  90
  91 #define CHANGE_BAR_WIDTH 70
  92 static void
  93 print_change_bar (double change, double max_change, int use_utf)
  94 {
  95     int units_per_cell = (int) ceil (max_change / CHANGE_BAR_WIDTH);
  96     static char const *ascii_boxes[8] = {
  97         "****","***" ,"***", "**",
  98         "**",  "*",   "*",   ""
  99     };
 100     static char const *utf_boxes[8] = {
 101         "█", "▉", "▊", "▋",
 102         "▌", "▍", "▎", "▏"
 103     };
 104     char const **boxes = use_utf ? utf_boxes : ascii_boxes;
 105
 106     /* For a 1.0x speedup we want a zero-size bar to show "no
 107      * change". */
 108     change -= 1.0;
 109
 110     while (change > units_per_cell) {
 111         printf ("%s", boxes[0]);
 112         change -= units_per_cell;
 113     }
 114
 115     change /= units_per_cell;
 116
 117     if (change > 7.5/8.0)
 118         printf ("%s", boxes[0]);
 119     else if (change > 6.5/8.0)
 120         printf ("%s", boxes[1]);
 121     else if (change > 5.5/8.0)
 122         printf ("%s", boxes[2]);
 123     else if (change > 4.5/8.0)
 124         printf ("%s", boxes[3]);
 125     else if (change > 3.5/8.0)
 126         printf ("%s", boxes[4]);
 127     else if (change > 2.5/8.0)
 128         printf ("%s", boxes[5]);
 129     else if (change > 1.5/8.0)
 130         printf ("%s", boxes[6]);
 131     else if (change > 0.5/8.0)
 132         printf ("%s", boxes[7]);
 133
 134     printf ("\n");
 135 }
 136
 137 static void
 138 test_diff_print_binary (test_diff_t                     *diff,
 139                         double                           max_change,
 140                         cairo_perf_report_options_t     *options)
 141 {
 142     printf ("%5s-%-4s %26s-%-3d  %6.2f %4.2f%% -> %6.2f %4.2f%%: %5.2fx ",
 143             diff->tests[0]->backend, diff->tests[0]->content,
 144             diff->tests[0]->name, diff->tests[0]->size,
 145             diff->tests[0]->stats.median_ticks / diff->tests[0]->stats.ticks_per_ms,
 146             diff->tests[0]->stats.std_dev * 100,
 147             diff->tests[1]->stats.median_ticks / diff->tests[1]->stats.ticks_per_ms,
 148             diff->tests[1]->stats.std_dev * 100,
 149             fabs (diff->change));
 150
 151     if (diff->change > 1.0)
 152         printf ("speedup\n");
 153     else
 154         printf ("slowdown\n");
 155
 156     if (options->print_change_bars)
 157         print_change_bar (fabs (diff->change), max_change,
 158                           options->use_utf);
 159 }
 160
 161 static void
 162 test_diff_print_multi (test_diff_t                      *diff,
 163                        double                            max_change,
 164                        cairo_perf_report_options_t      *options)
 165 {
 166     int i;
 167     double test_time;
 168     double change;
 169
 170     printf ("%s (backend: %s-%s, size: %d)\n",
 171             diff->tests[0]->name,
 172             diff->tests[0]->backend,
 173             diff->tests[0]->content,
 174             diff->tests[0]->size);
 175
 176     for (i = 0; i < diff->num_tests; i++) {
 177         test_time = diff->tests[i]->stats.min_ticks;
 178         if (options->use_ms)
 179             test_time /= diff->tests[i]->stats.ticks_per_ms;
 180         change = diff->max / test_time;
 181         printf ("%8s %6.2f: %5.2fx ",
 182                 diff->tests[i]->configuration,
 183                 diff->tests[i]->stats.min_ticks / diff->tests[i]->stats.ticks_per_ms,
 184                 change);
 185
 186         if (options->print_change_bars)
 187             print_change_bar (change, max_change, options->use_utf);
 188     }
 189
 190     printf("\n");
 191 }
 192
 193 #define MAX(a,b) ((a) > (b) ? (a) : (b))
 194 static void
 195 cairo_perf_reports_compare (cairo_perf_report_t         *reports,
 196                             int                          num_reports,
 197                             cairo_perf_report_options_t *options)
 198 {
 199     int i;
 200     test_report_t **tests, *min_test;
 201     test_diff_t *diff, *diffs;
 202     int num_diffs, max_diffs;
 203     double max_change;
 204     double test_time;
 205     int seen_non_null;
 206     cairo_bool_t printed_speedup = FALSE;
 207     cairo_bool_t printed_slowdown = FALSE;
 208
 209     assert (num_reports >= 2);
 210
 211     tests = xmalloc (num_reports * sizeof (test_report_t *));
 212
 213     max_diffs = reports[0].tests_count;
 214     for (i = 0; i < num_reports; i++) {
 215         tests[i] = reports[i].tests;
 216         if (reports[i].tests_count > max_diffs)
 217             max_diffs = reports[i].tests_count;
 218     }
 219
 220     diff = diffs = xmalloc (max_diffs * sizeof (test_diff_t));
 221
 222     num_diffs = 0;
 223     while (1) {
 224         /* We expect iterations values of 0 when multiple raw reports
 225          * for the same test have been condensed into the stats of the
 226          * first. So we just skip these later reports that have no
 227          * stats. */
 228         seen_non_null = 0;
 229         for (i = 0; i < num_reports; i++) {
 230             while (tests[i]->name && tests[i]->stats.iterations == 0)
 231                 tests[i]++;
 232             if (tests[i]->name)
 233                 seen_non_null++;
 234         }
 235
 236         if (seen_non_null < 2)
 237             break;
 238
 239         /* Find the minimum of all current tests, (we have to do this
 240          * in case some reports don't have a particular test). */
 241         for (i = 0; i < num_reports; i++) {
 242             if (tests[i]->name) {
 243                 min_test = tests[i];
 244                 break;
 245             }
 246         }
 247         for (++i; i < num_reports; i++) {
 248             if (tests[i]->name &&
 249                 test_report_cmp_backend_then_name (tests[i], min_test) < 0)
 250             {
 251                 min_test = tests[i];
 252             }
 253         }
 254
 255         /* For each report that has the current test, record it into
 256          * the diff structure. */
 257         diff->num_tests = 0;
 258         diff->tests = xmalloc (num_reports * sizeof (test_diff_t));
 259         for (i = 0; i < num_reports; i++) {
 260             if (tests[i]->name &&
 261                 test_report_cmp_backend_then_name (tests[i], min_test) == 0)
 262             {
 263                 test_time = tests[i]->stats.min_ticks;
 264                 if (options->use_ms)
 265                     test_time /= tests[i]->stats.ticks_per_ms;
 266                 if (diff->num_tests == 0) {
 267                     diff->min = test_time;
 268                     diff->max = test_time;
 269                 } else {
 270                     if (test_time < diff->min)
 271                         diff->min = test_time;
 272                     if (test_time > diff->max)
 273                         diff->max = test_time;
 274                 }
 275                 diff->tests[diff->num_tests++] = tests[i];
 276                 tests[i]++;
 277             }
 278         }
 279         diff->change = diff->max / diff->min;
 280
 281         if (num_reports == 2) {
 282             double old_time, new_time;
 283             if (diff->num_tests == 1) {
 284                 printf ("Only in %s: %s %s\n",
 285                         diff->tests[0]->configuration,
 286                         diff->tests[0]->backend,
 287                         diff->tests[0]->name);
 288                 continue;
 289             }
 290             old_time = diff->tests[0]->stats.min_ticks;
 291             new_time = diff->tests[1]->stats.min_ticks;
 292             if (options->use_ms) {
 293                 old_time /= diff->tests[0]->stats.ticks_per_ms;
 294                 new_time /= diff->tests[1]->stats.ticks_per_ms;
 295             }
 296             diff->change = old_time / new_time;
 297             if (diff->change < 1.0)
 298                 diff->change = - 1.0 / diff->change;
 299         }
 300
 301         diff++;
 302         num_diffs++;
 303     }
 304     if (num_diffs < 2)
 305         goto DONE;
 306
 307     if (num_reports == 2)
 308         qsort (diffs, num_diffs, sizeof (test_diff_t),
 309                test_diff_cmp_speedup_before_slowdown);
 310     else
 311         qsort (diffs, num_diffs, sizeof (test_diff_t), test_diff_cmp);
 312
 313     max_change = 1.0;
 314     for (i = 0; i < num_diffs; i++) {
 315         if (fabs (diffs[i].change) > max_change)
 316             max_change = fabs (diffs[i].change);
 317     }
 318
 319     if (num_reports == 2)
 320         printf ("old: %s\n"
 321                 "new: %s\n",
 322                 diffs->tests[0]->configuration,
 323                 diffs->tests[1]->configuration);
 324
 325     for (i = 0; i < num_diffs; i++) {
 326         diff = &diffs[i];
 327
 328         /* Discard as uninteresting a change which is less than the
 329          * minimum change required, (default may be overriden on
 330          * command-line). */
 331         if (fabs (diff->change) - 1.0 < options->min_change)
 332             continue;
 333
 334         if (num_reports == 2) {
 335             if (diff->change > 1.0 && ! printed_speedup) {
 336                 printf ("Speedups\n"
 337                         "========\n");
 338                 printed_speedup = TRUE;
 339             }
 340             if (diff->change < 1.0 && ! printed_slowdown) {
 341                 printf ("Slowdowns\n"
 342                         "=========\n");
 343                 printed_slowdown = TRUE;
 344             }
 345             test_diff_print_binary (diff, max_change, options);
 346         } else {
 347             test_diff_print_multi (diff, max_change, options);
 348         }
 349     }
 350
 351  DONE:
 352     for (i = 0; i < num_diffs; i++)
 353         free (diffs[i].tests);
 354     free (diffs);
 355     free (tests);
 356 }
 357
 358 static void
 359 usage (const char *argv0)
 360 {
 361     char const *basename = strrchr(argv0, '/');
 362     basename = basename ? basename+1 : argv0;
 363     fprintf (stderr,
 364              "Usage: %s [options] file1 file2 [...]\n\n",
 365              basename);
 366     fprintf (stderr,
 367              "Computes significant performance differences for cairo performance reports.\n"
 368              "Each file should be the output of the cairo-perf program (or \"make perf\").\n"
 369              "The following options are available:\n"
 370              "\n"
 371              "--no-utf    Use ascii stars instead of utf-8 change bars.\n"
 372              "            Four stars are printed per factor of speedup.\n"
 373              "\n"
 374              "--no-bars   Don't display change bars at all.\n\n"
 375              "\n"
 376              "--use-ms    Use milliseconds to calculate differences.\n"
 377              "            (instead of ticks which are hardware dependant)\n"
 378              "\n"
 379              "--min-change threshold[%%]\n"
 380              "            Suppress all changes below the given threshold.\n"
 381              "            The default threshold of 0.05 or 5%% ignores any\n"
 382              "            speedup or slowdown of 1.05 or less. A threshold\n"
 383              "            of 0 will cause all output to be reported.\n"
 384         );
 385     exit(1);
 386 }
 387
 388 static void
 389 parse_args(int                            argc,
 390            char const                   **argv,
 391            cairo_perf_diff_files_args_t  *args)
 392 {
 393     int i;
 394
 395     for (i = 1; i < argc; i++) {
 396         if (strcmp (argv[i], "--no-utf") == 0) {
 397             args->options.use_utf = 0;
 398         }
 399         else if (strcmp (argv[i], "--no-bars") == 0) {
 400             args->options.print_change_bars = 0;
 401         }
 402         else if (strcmp (argv[i], "--use-ms") == 0) {
 403             args->options.use_ms = 1;
 404         }
 405         else if (strcmp (argv[i], "--min-change") == 0) {
 406             char *end = NULL;
 407             i++;
 408             if (i >= argc)
 409                 usage (argv[0]);
 410             args->options.min_change = strtod (argv[i], &end);
 411             if (*end) {
 412                 if (*end == '%') {
 413                     args->options.min_change /= 100;
 414                 } else {
 415                     usage (argv[0]);
 416                 }
 417             }
 418         }
 419         else {
 420             args->num_filenames++;
 421             args->filenames = xrealloc (args->filenames,
 422                                         args->num_filenames * sizeof (char *));
 423             args->filenames[args->num_filenames - 1] = argv[i];
 424         }
 425     }
 426 }
 427
 428 int
 429 main (int argc, const char *argv[])
 430 {
 431     cairo_perf_diff_files_args_t args = {
 432         NULL,                   /* filenames */
 433         0,                      /* num_filenames */
 434         {
 435             0.05,               /* min change */
 436             1,                  /* use UTF-8? */
 437             1,                  /* display change bars? */
 438         }
 439     };
 440     cairo_perf_report_t *reports;
 441     test_report_t *t;
 442     int i;
 443
 444     parse_args (argc, argv, &args);
 445
 446     if (args.num_filenames < 2)
 447         usage (argv[0]);
 448
 449     reports = xmalloc (args.num_filenames * sizeof (cairo_perf_report_t));
 450
 451     for (i = 0; i < args.num_filenames; i++ )
 452         cairo_perf_report_load (&reports[i], args.filenames[i]);
 453
 454     cairo_perf_reports_compare (reports, args.num_filenames, &args.options);
 455
 456     /* Pointless memory cleanup, (would be a great place for talloc) */
 457     free (args.filenames);
 458     for (i = 0; i < args.num_filenames; i++) {
 459         for (t = reports[i].tests; t->name; t++) {
 460             free (t->samples);
 461             free (t->backend);
 462             free (t->name);
 463         }
 464         free (reports[i].tests);
 465         free (reports[i].configuration);
 466     }
 467     free (reports);
 468
 469     return 0;
 470 }