perf/cairo-perf-compare-backends.c

   1 /*
   2  * Copyright © 2006 Red Hat, Inc.
   3  * Copyright © 2009 Chris Wilson
   4  *
   5  * Permission to use, copy, modify, distribute, and sell this software
   6  * and its documentation for any purpose is hereby granted without
   7  * fee, provided that the above copyright notice appear in all copies
   8  * and that both that copyright notice and this permission notice
   9  * appear in supporting documentation, and that the name of the
  10  * copyright holders not be used in advertising or publicity
  11  * pertaining to distribution of the software without specific,
  12  * written prior permission. The copyright holders make no
  13  * representations about the suitability of this software for any
  14  * purpose.  It is provided "as is" without express or implied
  15  * warranty.
  16  *
  17  * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
  18  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  19  * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
  20  * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  21  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
  22  * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
  23  * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
  24  * SOFTWARE.
  25  *
  26  * Authors: Carl Worth <cworth@cworth.org>
  27  *          Chris Wilson <chris@chris-wilson.co.uk>
  28  */
  29
  30 #include "cairo-perf.h"
  31
  32 #include <stdio.h>
  33 #include <stdlib.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <ctype.h>
  37 #include <math.h>
  38 #include <assert.h>
  39
  40 typedef struct _cairo_perf_report_options {
  41     double min_change;
  42     int use_utf;
  43     int print_change_bars;
  44 } cairo_perf_report_options_t;
  45
  46 typedef struct _cairo_perf_diff_files_args {
  47     const char **filenames;
  48     int num_filenames;
  49     cairo_perf_report_options_t options;
  50 } cairo_perf_diff_files_args_t;
  51
  52 static int
  53 test_diff_cmp (const void *a, const void *b)
  54 {
  55     const test_diff_t *a_diff = a;
  56     const test_diff_t *b_diff = b;
  57
  58     /* Reverse sort by magnitude of change so larger changes come
  59      * first */
  60     if (a_diff->change > b_diff->change)
  61         return -1;
  62
  63     if (a_diff->change < b_diff->change)
  64         return 1;
  65
  66     return 0;
  67 }
  68
  69 #define CHANGE_BAR_WIDTH 70
  70 static void
  71 print_change_bar (double change, double max_change, int use_utf)
  72 {
  73     int units_per_cell = (int) ceil (max_change / CHANGE_BAR_WIDTH);
  74     static char const *ascii_boxes[8] = {
  75         "****","***" ,"***", "**",
  76         "**",  "*",   "*",   ""
  77     };
  78     static char const *utf_boxes[8] = {
  79         "█", "▉", "▊", "▋",
  80         "▌", "▍", "▎", "▏"
  81     };
  82     char const **boxes = use_utf ? utf_boxes : ascii_boxes;
  83
  84     /* For a 1.0x speedup we want a zero-size bar to show "no
  85      * change". */
  86     change -= 1.0;
  87
  88     while (change > units_per_cell) {
  89         printf ("%s", boxes[0]);
  90         change -= units_per_cell;
  91     }
  92
  93     change /= units_per_cell;
  94
  95     if (change > 7.5/8.0)
  96         printf ("%s", boxes[0]);
  97     else if (change > 6.5/8.0)
  98         printf ("%s", boxes[1]);
  99     else if (change > 5.5/8.0)
 100         printf ("%s", boxes[2]);
 101     else if (change > 4.5/8.0)
 102         printf ("%s", boxes[3]);
 103     else if (change > 3.5/8.0)
 104         printf ("%s", boxes[4]);
 105     else if (change > 2.5/8.0)
 106         printf ("%s", boxes[5]);
 107     else if (change > 1.5/8.0)
 108         printf ("%s", boxes[6]);
 109     else if (change > 0.5/8.0)
 110         printf ("%s", boxes[7]);
 111
 112     printf ("\n");
 113 }
 114
 115 static void
 116 test_diff_print (test_diff_t                    *diff,
 117                  double                          max_change,
 118                  cairo_perf_report_options_t    *options)
 119 {
 120     int i;
 121     double test_time;
 122     double change;
 123
 124     if (diff->tests[0]->size != 0) {
 125         printf ("(%s, size: %d)\n",
 126                 diff->tests[0]->name,
 127                 diff->tests[0]->size);
 128     } else {
 129         printf ("(%s)\n", diff->tests[0]->name);
 130     }
 131
 132     for (i = 0; i < diff->num_tests; i++) {
 133         test_time = diff->tests[i]->stats.min_ticks;
 134         test_time /= diff->tests[i]->stats.ticks_per_ms;
 135         change = diff->max / test_time;
 136         printf ("%8s-%s-%s\t%6.2f: %5.2fx ",
 137                 diff->tests[i]->backend,
 138                 diff->tests[i]->content,
 139                 diff->tests[i]->configuration,
 140                 diff->tests[i]->stats.min_ticks / diff->tests[i]->stats.ticks_per_ms,
 141                 change);
 142
 143         if (options->print_change_bars)
 144             print_change_bar (change, max_change, options->use_utf);
 145     }
 146
 147     printf("\n");
 148 }
 149
 150 #define MAX(a,b) ((a) > (b) ? (a) : (b))
 151 static void
 152 cairo_perf_reports_compare (cairo_perf_report_t         *reports,
 153                             int                          num_reports,
 154                             cairo_perf_report_options_t *options)
 155 {
 156     int i;
 157     test_report_t **tests, *min_test;
 158     test_diff_t *diff, *diffs;
 159     int num_diffs, max_diffs;
 160     double max_change;
 161     double test_time;
 162     int seen_non_null;
 163
 164     tests = xmalloc (num_reports * sizeof (test_report_t *));
 165
 166     max_diffs = reports[0].tests_count;
 167     for (i = 0; i < num_reports; i++) {
 168         tests[i] = reports[i].tests;
 169         if (reports[i].tests_count > max_diffs)
 170             max_diffs = reports[i].tests_count;
 171     }
 172
 173     diff = diffs = xmalloc (max_diffs * sizeof (test_diff_t));
 174
 175     num_diffs = 0;
 176     while (1) {
 177         int num_tests;
 178
 179         /* We expect iterations values of 0 when multiple raw reports
 180          * for the same test have been condensed into the stats of the
 181          * first. So we just skip these later reports that have no
 182          * stats. */
 183         seen_non_null = 0;
 184         for (i = 0; i < num_reports; i++) {
 185             while (tests[i]->name && tests[i]->stats.iterations == 0)
 186                 tests[i]++;
 187             if (tests[i]->name)
 188                 seen_non_null++;
 189         }
 190         if (! seen_non_null)
 191             break;
 192
 193         /* Find the minimum of all current tests, (we have to do this
 194          * in case some reports don't have a particular test). */
 195         for (i = 0; i < num_reports; i++) {
 196             if (tests[i]->name) {
 197                 min_test = tests[i];
 198                 break;
 199             }
 200         }
 201         for (++i; i < num_reports; i++) {
 202             if (tests[i]->name && test_report_cmp_name (tests[i], min_test) < 0)
 203                 min_test = tests[i];
 204         }
 205
 206         num_tests = 0;
 207         for (i = 0; i < num_reports; i++) {
 208             test_report_t *test;
 209             int n = 0;
 210
 211             test = tests[i];
 212             while (test[n].name &&
 213                     test_report_cmp_name (&test[n], min_test) == 0)
 214             {
 215                 n++;
 216             }
 217
 218             num_tests += n;
 219         }
 220
 221         /* For each report that has the current test, record it into
 222          * the diff structure. */
 223         diff->num_tests = 0;
 224         diff->tests = xmalloc (num_tests * sizeof (test_diff_t));
 225         for (i = 0; i < num_reports; i++) {
 226             while (tests[i]->name &&
 227                     test_report_cmp_name (tests[i], min_test) == 0)
 228             {
 229                 test_time = tests[i]->stats.min_ticks;
 230                 if (test_time > 0) {
 231                     test_time /= tests[i]->stats.ticks_per_ms;
 232                     if (diff->num_tests == 0) {
 233                         diff->min = test_time;
 234                         diff->max = test_time;
 235                     } else {
 236                         if (test_time < diff->min)
 237                             diff->min = test_time;
 238                         if (test_time > diff->max)
 239                             diff->max = test_time;
 240                     }
 241                     diff->tests[diff->num_tests++] = tests[i];
 242                 }
 243                 tests[i]++;
 244             }
 245         }
 246         diff->change = diff->max / diff->min;
 247
 248         diff++;
 249         num_diffs++;
 250     }
 251     if (num_diffs == 0)
 252         goto DONE;
 253
 254     qsort (diffs, num_diffs, sizeof (test_diff_t), test_diff_cmp);
 255
 256     max_change = 1.0;
 257     for (i = 0; i < num_diffs; i++) {
 258         if (fabs (diffs[i].change) > max_change)
 259             max_change = fabs (diffs[i].change);
 260     }
 261
 262     for (i = 0; i < num_diffs; i++) {
 263         diff = &diffs[i];
 264
 265         /* Discard as uninteresting a change which is less than the
 266          * minimum change required, (default may be overriden on
 267          * command-line). */
 268         if (fabs (diff->change) - 1.0 < options->min_change)
 269             continue;
 270
 271         test_diff_print (diff, max_change, options);
 272     }
 273
 274     for (i = 0; i < num_diffs; i++)
 275         free (diffs[i].tests);
 276  DONE:
 277     free (diffs);
 278     free (tests);
 279 }
 280
 281 static void
 282 usage (const char *argv0)
 283 {
 284     char const *basename = strrchr(argv0, '/');
 285     basename = basename ? basename+1 : argv0;
 286     fprintf (stderr,
 287              "Usage: %s [options] file [...]\n\n",
 288              basename);
 289     fprintf (stderr,
 290              "Computes significant performance differences for cairo performance reports.\n"
 291              "Each file should be the output of the cairo-perf program (or \"make perf\").\n"
 292              "The following options are available:\n"
 293              "\n"
 294              "--no-utf    Use ascii stars instead of utf-8 change bars.\n"
 295              "            Four stars are printed per factor of speedup.\n"
 296              "\n"
 297              "--no-bars   Don't display change bars at all.\n\n"
 298              "\n"
 299              "--use-ms    Use milliseconds to calculate differences.\n"
 300              "            (instead of ticks which are hardware dependant)\n"
 301              "\n"
 302              "--min-change threshold[%%]\n"
 303              "            Suppress all changes below the given threshold.\n"
 304              "            The default threshold of 0.05 or 5%% ignores any\n"
 305              "            speedup or slowdown of 1.05 or less. A threshold\n"
 306              "            of 0 will cause all output to be reported.\n"
 307         );
 308     exit(1);
 309 }
 310
 311 static void
 312 parse_args(int                            argc,
 313            char const                   **argv,
 314            cairo_perf_diff_files_args_t  *args)
 315 {
 316     int i;
 317
 318     for (i = 1; i < argc; i++) {
 319         if (strcmp (argv[i], "--no-utf") == 0) {
 320             args->options.use_utf = 0;
 321         }
 322         else if (strcmp (argv[i], "--no-bars") == 0) {
 323             args->options.print_change_bars = 0;
 324         }
 325         else if (strcmp (argv[i], "--min-change") == 0) {
 326             char *end = NULL;
 327             i++;
 328             if (i >= argc)
 329                 usage (argv[0]);
 330             args->options.min_change = strtod (argv[i], &end);
 331             if (*end) {
 332                 if (*end == '%') {
 333                     args->options.min_change /= 100;
 334                 } else {
 335                     usage (argv[0]);
 336                 }
 337             }
 338         }
 339         else {
 340             args->num_filenames++;
 341             args->filenames = xrealloc (args->filenames,
 342                                         args->num_filenames * sizeof (char *));
 343             args->filenames[args->num_filenames - 1] = argv[i];
 344         }
 345     }
 346 }
 347
 348 int
 349 main (int argc, const char *argv[])
 350 {
 351     cairo_perf_diff_files_args_t args = {
 352         NULL,                   /* filenames */
 353         0,                      /* num_filenames */
 354         {
 355             0.05,               /* min change */
 356             1,                  /* use UTF-8? */
 357             1,                  /* display change bars? */
 358         }
 359     };
 360     cairo_perf_report_t *reports;
 361     test_report_t *t;
 362     int i;
 363
 364     parse_args (argc, argv, &args);
 365
 366     if (args.num_filenames < 1)
 367         usage (argv[0]);
 368
 369     reports = xcalloc (args.num_filenames, sizeof (cairo_perf_report_t));
 370
 371     for (i = 0; i < args.num_filenames; i++) {
 372         cairo_perf_report_load (&reports[i], args.filenames[i],
 373                                 test_report_cmp_name);
 374         printf ("loaded: %s, %d tests\n",
 375                 args.filenames[i], reports[i].tests_count);
 376     }
 377
 378     cairo_perf_reports_compare (reports, args.num_filenames, &args.options);
 379
 380     /* Pointless memory cleanup, (would be a great place for talloc) */
 381     free (args.filenames);
 382     for (i = 0; i < args.num_filenames; i++) {
 383         for (t = reports[i].tests; t->name; t++) {
 384             free (t->samples);
 385             free (t->backend);
 386             free (t->name);
 387         }
 388         free (reports[i].tests);
 389         free (reports[i].configuration);
 390     }
 391     free (reports);
 392
 393     return 0;
 394 }