compiler-rt/lib/profile/GCDAProfiling.c

   1 /*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 |*===----------------------------------------------------------------------===*|
   8 |*
   9 |* This file implements the call back routines for the gcov profiling
  10 |* instrumentation pass. Link against this library when running code through
  11 |* the -insert-gcov-profiling LLVM pass.
  12 |*
  13 |* We emit files in a corrupt version of GCOV's "gcda" file format. These files
  14 |* are only close enough that LCOV will happily parse them. Anything that lcov
  15 |* ignores is missing.
  16 |*
  17 |* TODO: gcov is multi-process safe by having each exit open the existing file
  18 |* and append to it. We'd like to achieve that and be thread-safe too.
  19 |*
  20 \*===----------------------------------------------------------------------===*/
  21
  22 #if !defined(__Fuchsia__)
  23
  24 #include <errno.h>
  25 #include <fcntl.h>
  26 #include <stdint.h>
  27 #include <stdio.h>
  28 #include <stdlib.h>
  29 #include <string.h>
  30
  31 #if defined(_WIN32)
  32 #define WIN32_LEAN_AND_MEAN
  33 #include <windows.h>
  34 #include "WindowsMMap.h"
  35 #else
  36 #include <sys/file.h>
  37 #include <sys/mman.h>
  38 #include <sys/types.h>
  39 #include <unistd.h>
  40 #endif
  41
  42 #include "InstrProfiling.h"
  43 #include "InstrProfilingUtil.h"
  44
  45 /* #define DEBUG_GCDAPROFILING */
  46
  47 enum {
  48   GCOV_DATA_MAGIC = 0x67636461, // "gcda"
  49
  50   GCOV_TAG_FUNCTION = 0x01000000,
  51   GCOV_TAG_COUNTER_ARCS = 0x01a10000,
  52   // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9.
  53   GCOV_TAG_OBJECT_SUMMARY = 0xa1000000,
  54   GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000,
  55 };
  56
  57 /*
  58  * --- GCOV file format I/O primitives ---
  59  */
  60
  61 /*
  62  * The current file name we're outputting. Used primarily for error logging.
  63  */
  64 static char *filename = NULL;
  65
  66 /*
  67  * The current file we're outputting.
  68  */
  69 static FILE *output_file = NULL;
  70
  71 /*
  72  * Buffer that we write things into.
  73  */
  74 #define WRITE_BUFFER_SIZE (128 * 1024)
  75 static unsigned char *write_buffer = NULL;
  76 static uint64_t cur_buffer_size = 0;
  77 static uint64_t cur_pos = 0;
  78 static uint64_t file_size = 0;
  79 static int new_file = 0;
  80 static int gcov_version;
  81 #if defined(_WIN32)
  82 static HANDLE mmap_handle = NULL;
  83 #endif
  84 static int fd = -1;
  85
  86 typedef void (*fn_ptr)(void);
  87
  88 typedef void* dynamic_object_id;
  89 // The address of this variable identifies a given dynamic object.
  90 static dynamic_object_id current_id;
  91 #define CURRENT_ID (&current_id)
  92
  93 struct fn_node {
  94   dynamic_object_id id;
  95   fn_ptr fn;
  96   struct fn_node* next;
  97 };
  98
  99 struct fn_list {
 100   struct fn_node *head, *tail;
 101 };
 102
 103 /*
 104  * A list of functions to write out the data, shared between all dynamic objects.
 105  */
 106 struct fn_list writeout_fn_list;
 107
 108 /*
 109  *  A list of reset functions, shared between all dynamic objects.
 110  */
 111 struct fn_list reset_fn_list;
 112
 113 static void fn_list_insert(struct fn_list* list, fn_ptr fn) {
 114   struct fn_node* new_node = malloc(sizeof(struct fn_node));
 115   new_node->fn = fn;
 116   new_node->next = NULL;
 117   new_node->id = CURRENT_ID;
 118
 119   if (!list->head) {
 120     list->head = list->tail = new_node;
 121   } else {
 122     list->tail->next = new_node;
 123     list->tail = new_node;
 124   }
 125 }
 126
 127 static void fn_list_remove(struct fn_list* list) {
 128   struct fn_node* curr = list->head;
 129   struct fn_node* prev = NULL;
 130   struct fn_node* next = NULL;
 131
 132   while (curr) {
 133     next = curr->next;
 134
 135     if (curr->id == CURRENT_ID) {
 136       if (curr == list->head) {
 137         list->head = next;
 138       }
 139
 140       if (curr == list->tail) {
 141         list->tail = prev;
 142       }
 143
 144       if (prev) {
 145         prev->next = next;
 146       }
 147
 148       free(curr);
 149     } else {
 150       prev = curr;
 151     }
 152
 153     curr = next;
 154   }
 155 }
 156
 157 static void resize_write_buffer(uint64_t size) {
 158   if (!new_file) return;
 159   size += cur_pos;
 160   if (size <= cur_buffer_size) return;
 161   size = (size - 1) / WRITE_BUFFER_SIZE + 1;
 162   size *= WRITE_BUFFER_SIZE;
 163   write_buffer = realloc(write_buffer, size);
 164   cur_buffer_size = size;
 165 }
 166
 167 static void write_bytes(const char *s, size_t len) {
 168   resize_write_buffer(len);
 169   memcpy(&write_buffer[cur_pos], s, len);
 170   cur_pos += len;
 171 }
 172
 173 static void write_32bit_value(uint32_t i) {
 174   write_bytes((char*)&i, 4);
 175 }
 176
 177 static void write_64bit_value(uint64_t i) {
 178   // GCOV uses a lo-/hi-word format even on big-endian systems.
 179   // See also GCOVBuffer::readInt64 in LLVM.
 180   uint32_t lo = (uint32_t) i;
 181   uint32_t hi = (uint32_t) (i >> 32);
 182   write_32bit_value(lo);
 183   write_32bit_value(hi);
 184 }
 185
 186 static uint32_t read_32bit_value(void) {
 187   uint32_t val;
 188
 189   if (new_file)
 190     return (uint32_t)-1;
 191
 192   val = *(uint32_t*)&write_buffer[cur_pos];
 193   cur_pos += 4;
 194   return val;
 195 }
 196
 197 static uint64_t read_64bit_value(void) {
 198   // GCOV uses a lo-/hi-word format even on big-endian systems.
 199   // See also GCOVBuffer::readInt64 in LLVM.
 200   uint32_t lo = read_32bit_value();
 201   uint32_t hi = read_32bit_value();
 202   return ((uint64_t)hi << 32) | ((uint64_t)lo);
 203 }
 204
 205 static char *mangle_filename(const char *orig_filename) {
 206   char *new_filename;
 207   size_t prefix_len;
 208   int prefix_strip;
 209   const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len);
 210
 211   if (prefix == NULL)
 212     return strdup(orig_filename);
 213
 214   new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1);
 215   lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len,
 216                        prefix_strip);
 217
 218   return new_filename;
 219 }
 220
 221 static int map_file(void) {
 222   fseek(output_file, 0L, SEEK_END);
 223   file_size = ftell(output_file);
 224
 225   /* A size of 0 means the file has been created just now (possibly by another
 226    * process in lock-after-open race condition). No need to mmap. */
 227   if (file_size == 0)
 228     return -1;
 229
 230 #if defined(_WIN32)
 231   HANDLE mmap_fd;
 232   if (fd == -1)
 233     mmap_fd = INVALID_HANDLE_VALUE;
 234   else
 235     mmap_fd = (HANDLE)_get_osfhandle(fd);
 236
 237   mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL);
 238   if (mmap_handle == NULL) {
 239     fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n",
 240             filename, GetLastError());
 241     return -1;
 242   }
 243
 244   write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size);
 245   if (write_buffer == NULL) {
 246     fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename,
 247             GetLastError());
 248     CloseHandle(mmap_handle);
 249     return -1;
 250   }
 251 #else
 252   write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE,
 253                       MAP_FILE | MAP_SHARED, fd, 0);
 254   if (write_buffer == (void *)-1) {
 255     int errnum = errno;
 256     fprintf(stderr, "profiling: %s: cannot map: %s\n", filename,
 257             strerror(errnum));
 258     return -1;
 259   }
 260 #endif
 261
 262   return 0;
 263 }
 264
 265 static void unmap_file(void) {
 266 #if defined(_WIN32)
 267   if (!UnmapViewOfFile(write_buffer)) {
 268     fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
 269             GetLastError());
 270   }
 271
 272   if (!CloseHandle(mmap_handle)) {
 273     fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n",
 274             filename, GetLastError());
 275   }
 276
 277   mmap_handle = NULL;
 278 #else
 279   if (munmap(write_buffer, file_size) == -1) {
 280     int errnum = errno;
 281     fprintf(stderr, "profiling: %s: cannot munmap: %s\n", filename,
 282             strerror(errnum));
 283   }
 284 #endif
 285
 286   write_buffer = NULL;
 287   file_size = 0;
 288 }
 289
 290 /*
 291  * --- LLVM line counter API ---
 292  */
 293
 294 /* A file in this case is a translation unit. Each .o file built with line
 295  * profiling enabled will emit to a different file. Only one file may be
 296  * started at a time.
 297  */
 298 COMPILER_RT_VISIBILITY
 299 void llvm_gcda_start_file(const char *orig_filename, uint32_t version,
 300                           uint32_t checksum) {
 301   const char *mode = "r+b";
 302   filename = mangle_filename(orig_filename);
 303
 304   /* Try just opening the file. */
 305   fd = open(filename, O_RDWR | O_BINARY);
 306
 307   if (fd == -1) {
 308     /* Try creating the file. */
 309     fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
 310     if (fd != -1) {
 311       mode = "w+b";
 312     } else {
 313       /* Try creating the directories first then opening the file. */
 314       __llvm_profile_recursive_mkdir(filename);
 315       fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
 316       if (fd != -1) {
 317         mode = "w+b";
 318       } else {
 319         /* Another process may have created the file just now.
 320          * Try opening it without O_CREAT and O_EXCL. */
 321         fd = open(filename, O_RDWR | O_BINARY);
 322         if (fd == -1) {
 323           /* Bah! It's hopeless. */
 324           int errnum = errno;
 325           fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
 326                   strerror(errnum));
 327           return;
 328         }
 329       }
 330     }
 331   }
 332
 333   /* Try to flock the file to serialize concurrent processes writing out to the
 334    * same GCDA. This can fail if the filesystem doesn't support it, but in that
 335    * case we'll just carry on with the old racy behaviour and hope for the best.
 336    */
 337   lprofLockFd(fd);
 338   output_file = fdopen(fd, mode);
 339
 340   /* Initialize the write buffer. */
 341   new_file = 0;
 342   write_buffer = NULL;
 343   cur_buffer_size = 0;
 344   cur_pos = 0;
 345
 346   if (map_file() == -1) {
 347     /* The file has been created just now (file_size == 0) or mmap failed
 348      * unexpectedly. In the latter case, try to recover by clobbering. */
 349     new_file = 1;
 350     write_buffer = NULL;
 351     resize_write_buffer(WRITE_BUFFER_SIZE);
 352     memset(write_buffer, 0, WRITE_BUFFER_SIZE);
 353   }
 354
 355   /* gcda file, version, stamp checksum. */
 356   {
 357     uint8_t c3 = version >> 24;
 358     uint8_t c2 = (version >> 16) & 255;
 359     uint8_t c1 = (version >> 8) & 255;
 360     gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
 361                              : (c3 - '0') * 10 + c1 - '0';
 362   }
 363   write_32bit_value(GCOV_DATA_MAGIC);
 364   write_32bit_value(version);
 365   write_32bit_value(checksum);
 366
 367 #ifdef DEBUG_GCDAPROFILING
 368   fprintf(stderr, "llvmgcda: [%s]\n", orig_filename);
 369 #endif
 370 }
 371
 372 COMPILER_RT_VISIBILITY
 373 void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
 374                              uint32_t cfg_checksum) {
 375   uint32_t len = 2;
 376   int use_extra_checksum = gcov_version >= 47;
 377
 378   if (use_extra_checksum)
 379     len++;
 380 #ifdef DEBUG_GCDAPROFILING
 381   fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident);
 382 #endif
 383   if (!output_file) return;
 384
 385   /* function tag */
 386   write_32bit_value(GCOV_TAG_FUNCTION);
 387   write_32bit_value(len);
 388   write_32bit_value(ident);
 389   write_32bit_value(func_checksum);
 390   if (use_extra_checksum)
 391     write_32bit_value(cfg_checksum);
 392 }
 393
 394 COMPILER_RT_VISIBILITY
 395 void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
 396   uint32_t i;
 397   uint64_t *old_ctrs = NULL;
 398   uint32_t val = 0;
 399   uint64_t save_cur_pos = cur_pos;
 400
 401   if (!output_file) return;
 402
 403   val = read_32bit_value();
 404
 405   if (val != (uint32_t)-1) {
 406     /* There are counters present in the file. Merge them. */
 407     if (val != GCOV_TAG_COUNTER_ARCS) {
 408       fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
 409                       "corrupt arc tag (0x%08x)\n",
 410               filename, val);
 411       return;
 412     }
 413
 414     val = read_32bit_value();
 415     if (val == (uint32_t)-1 || val / 2 != num_counters) {
 416       fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
 417                       "mismatched number of counters (%d)\n",
 418               filename, val);
 419       return;
 420     }
 421
 422     old_ctrs = malloc(sizeof(uint64_t) * num_counters);
 423     for (i = 0; i < num_counters; ++i)
 424       old_ctrs[i] = read_64bit_value();
 425   }
 426
 427   cur_pos = save_cur_pos;
 428
 429   /* Counter #1 (arcs) tag */
 430   write_32bit_value(GCOV_TAG_COUNTER_ARCS);
 431   write_32bit_value(num_counters * 2);
 432   for (i = 0; i < num_counters; ++i) {
 433     counters[i] += (old_ctrs ? old_ctrs[i] : 0);
 434     write_64bit_value(counters[i]);
 435   }
 436
 437   free(old_ctrs);
 438
 439 #ifdef DEBUG_GCDAPROFILING
 440   fprintf(stderr, "llvmgcda:   %u arcs\n", num_counters);
 441   for (i = 0; i < num_counters; ++i)
 442     fprintf(stderr, "llvmgcda:   %llu\n", (unsigned long long)counters[i]);
 443 #endif
 444 }
 445
 446 COMPILER_RT_VISIBILITY
 447 void llvm_gcda_summary_info(void) {
 448   uint32_t runs = 1;
 449   static uint32_t run_counted = 0; // We only want to increase the run count once.
 450   uint32_t val = 0;
 451   uint64_t save_cur_pos = cur_pos;
 452
 453   if (!output_file) return;
 454
 455   val = read_32bit_value();
 456
 457   if (val != (uint32_t)-1) {
 458     /* There are counters present in the file. Merge them. */
 459     uint32_t gcov_tag =
 460         gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY;
 461     if (val != gcov_tag) {
 462       fprintf(stderr,
 463               "profiling: %s: cannot merge previous run count: "
 464               "corrupt object tag (0x%08x)\n",
 465               filename, val);
 466       return;
 467     }
 468
 469     val = read_32bit_value(); /* length */
 470     uint32_t prev_runs;
 471     if (gcov_version < 90) {
 472       read_32bit_value();
 473       read_32bit_value();
 474       prev_runs = read_32bit_value();
 475     } else {
 476       prev_runs = read_32bit_value();
 477       read_32bit_value();
 478     }
 479     for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i)
 480       read_32bit_value();
 481     /* Add previous run count to new counter, if not already counted before. */
 482     runs = run_counted ? prev_runs : prev_runs + 1;
 483   }
 484
 485   cur_pos = save_cur_pos;
 486
 487   if (gcov_version >= 90) {
 488     write_32bit_value(GCOV_TAG_OBJECT_SUMMARY);
 489     write_32bit_value(2);
 490     write_32bit_value(runs);
 491     write_32bit_value(0); // sum_max
 492   } else {
 493     // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set
 494     // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which
 495     // can make gcov read "Runs:".
 496     write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY);
 497     write_32bit_value(3);
 498     write_32bit_value(0);
 499     write_32bit_value(0);
 500     write_32bit_value(runs);
 501   }
 502
 503   run_counted = 1;
 504
 505 #ifdef DEBUG_GCDAPROFILING
 506   fprintf(stderr, "llvmgcda:   %u runs\n", runs);
 507 #endif
 508 }
 509
 510 COMPILER_RT_VISIBILITY
 511 void llvm_gcda_end_file(void) {
 512   /* Write out EOF record. */
 513   if (output_file) {
 514     write_bytes("\0\0\0\0\0\0\0\0", 8);
 515
 516     if (new_file) {
 517       fwrite(write_buffer, cur_pos, 1, output_file);
 518       free(write_buffer);
 519     } else {
 520       unmap_file();
 521     }
 522
 523     fflush(output_file);
 524     lprofUnlockFd(fd);
 525     fclose(output_file);
 526     output_file = NULL;
 527     write_buffer = NULL;
 528   }
 529   free(filename);
 530
 531 #ifdef DEBUG_GCDAPROFILING
 532   fprintf(stderr, "llvmgcda: -----\n");
 533 #endif
 534 }
 535
 536 COMPILER_RT_VISIBILITY
 537 void llvm_register_writeout_function(fn_ptr fn) {
 538   fn_list_insert(&writeout_fn_list, fn);
 539 }
 540
 541 COMPILER_RT_VISIBILITY
 542 void llvm_writeout_files(void) {
 543   struct fn_node *curr = writeout_fn_list.head;
 544
 545   while (curr) {
 546     if (curr->id == CURRENT_ID) {
 547       curr->fn();
 548     }
 549     curr = curr->next;
 550   }
 551 }
 552
 553 #ifndef _WIN32
 554 // __attribute__((destructor)) and destructors whose priorities are greater than
 555 // 100 run before this function and can thus be tracked. The priority is
 556 // compatible with GCC 7 onwards.
 557 #if __GNUC__ >= 9
 558 #pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
 559 #endif
 560 __attribute__((destructor(100)))
 561 #endif
 562 static void llvm_writeout_and_clear(void) {
 563   llvm_writeout_files();
 564   fn_list_remove(&writeout_fn_list);
 565 }
 566
 567 COMPILER_RT_VISIBILITY
 568 void llvm_register_reset_function(fn_ptr fn) {
 569   fn_list_insert(&reset_fn_list, fn);
 570 }
 571
 572 COMPILER_RT_VISIBILITY
 573 void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); }
 574
 575 COMPILER_RT_VISIBILITY
 576 void llvm_reset_counters(void) {
 577   struct fn_node *curr = reset_fn_list.head;
 578
 579   while (curr) {
 580     if (curr->id == CURRENT_ID) {
 581       curr->fn();
 582     }
 583     curr = curr->next;
 584   }
 585 }
 586
 587 #if !defined(_WIN32) && !defined(__wasm__)
 588 COMPILER_RT_VISIBILITY
 589 pid_t __gcov_fork() {
 590   pid_t parent_pid = getpid();
 591   pid_t pid = fork();
 592
 593   if (pid == 0) {
 594     pid_t child_pid = getpid();
 595     if (child_pid != parent_pid) {
 596       // The pid changed so we've a fork (one could have its own fork function)
 597       // Just reset the counters for this child process
 598       // threads.
 599       llvm_reset_counters();
 600     }
 601   }
 602   return pid;
 603 }
 604 #endif
 605
 606 COMPILER_RT_VISIBILITY
 607 void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) {
 608   static int atexit_ran = 0;
 609
 610   if (wfn)
 611     llvm_register_writeout_function(wfn);
 612
 613   if (rfn)
 614     llvm_register_reset_function(rfn);
 615
 616   if (atexit_ran == 0) {
 617     atexit_ran = 1;
 618
 619     /* Make sure we write out the data and delete the data structures. */
 620     lprofAtExit(llvm_delete_reset_function_list);
 621 #ifdef _WIN32
 622     lprofAtExit(llvm_writeout_and_clear);
 623 #endif
 624   }
 625 }
 626
 627 #if defined(_AIX)
 628 COMPILER_RT_VISIBILITY __attribute__((constructor)) void
 629 __llvm_profile_gcov_initialize() {
 630   const __llvm_gcov_init_func_struct *InitFuncStart =
 631       __llvm_profile_begin_covinit();
 632   const __llvm_gcov_init_func_struct *InitFuncEnd =
 633       __llvm_profile_end_covinit();
 634
 635   for (const __llvm_gcov_init_func_struct *Ptr = InitFuncStart;
 636        Ptr != InitFuncEnd; ++Ptr) {
 637     fn_ptr wfn = (fn_ptr)Ptr->WriteoutFunction;
 638     fn_ptr rfn = (fn_ptr)Ptr->ResetFunction;
 639     if (!(wfn && rfn))
 640       continue;
 641     llvm_gcov_init(wfn, rfn);
 642   }
 643 }
 644 #endif
 645
 646 void __gcov_dump(void) {
 647   for (struct fn_node *f = writeout_fn_list.head; f; f = f->next)
 648     f->fn();
 649 }
 650
 651 void __gcov_reset(void) {
 652   for (struct fn_node *f = reset_fn_list.head; f; f = f->next)
 653     f->fn();
 654 }
 655
 656 #endif