compiler-rt/lib/profile/InstrProfilingMerge.c

   1 /*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 |*===----------------------------------------------------------------------===*
   8 |* This file defines the API needed for in-process merging of profile data
   9 |* stored in memory buffer.
  10 \*===---------------------------------------------------------------------===*/
  11
  12 #include "InstrProfiling.h"
  13 #include "InstrProfilingInternal.h"
  14 #include "InstrProfilingUtil.h"
  15
  16 #define INSTR_PROF_VALUE_PROF_DATA
  17 #include "profile/InstrProfData.inc"
  18
  19 COMPILER_RT_VISIBILITY
  20 void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
  21
  22 COMPILER_RT_VISIBILITY
  23 uint64_t lprofGetLoadModuleSignature(void) {
  24   /* A very fast way to compute a module signature.  */
  25   uint64_t Version = __llvm_profile_get_version();
  26   uint64_t NumCounters = __llvm_profile_get_num_counters(
  27       __llvm_profile_begin_counters(), __llvm_profile_end_counters());
  28   uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
  29                                                  __llvm_profile_end_data());
  30   uint64_t NamesSize =
  31       (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
  32   uint64_t NumVnodes =
  33       (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
  34   const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
  35
  36   return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
  37          (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
  38          __llvm_profile_get_magic();
  39 }
  40
  41 #ifdef __GNUC__
  42 #pragma GCC diagnostic push
  43 #pragma GCC diagnostic ignored "-Wcast-qual"
  44 #elif defined(__clang__)
  45 #pragma clang diagnostic push
  46 #pragma clang diagnostic ignored "-Wcast-qual"
  47 #endif
  48
  49 /* Returns 1 if profile is not structurally compatible.  */
  50 COMPILER_RT_VISIBILITY
  51 int __llvm_profile_check_compatibility(const char *ProfileData,
  52                                        uint64_t ProfileSize) {
  53   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
  54   __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
  55   SrcDataStart =
  56       (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
  57                               Header->BinaryIdsSize);
  58   SrcDataEnd = SrcDataStart + Header->NumData;
  59
  60   if (ProfileSize < sizeof(__llvm_profile_header))
  61     return 1;
  62
  63   /* Check the header first.  */
  64   if (Header->Magic != __llvm_profile_get_magic() ||
  65       Header->Version != __llvm_profile_get_version() ||
  66       Header->NumData !=
  67           __llvm_profile_get_num_data(__llvm_profile_begin_data(),
  68                                       __llvm_profile_end_data()) ||
  69       Header->NumCounters !=
  70           __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
  71                                           __llvm_profile_end_counters()) ||
  72       Header->NumBitmapBytes !=
  73           __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
  74                                               __llvm_profile_end_bitmap()) ||
  75       Header->NamesSize !=
  76           __llvm_profile_get_name_size(__llvm_profile_begin_names(),
  77                                        __llvm_profile_end_names()) ||
  78       Header->ValueKindLast != IPVK_Last)
  79     return 1;
  80
  81   if (ProfileSize <
  82       sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
  83           Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
  84           Header->NumCounters * __llvm_profile_counter_entry_size() +
  85           Header->NumBitmapBytes)
  86     return 1;
  87
  88   for (SrcData = SrcDataStart,
  89        DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
  90        SrcData < SrcDataEnd; ++SrcData, ++DstData) {
  91     if (SrcData->NameRef != DstData->NameRef ||
  92         SrcData->FuncHash != DstData->FuncHash ||
  93         SrcData->NumCounters != DstData->NumCounters ||
  94         SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
  95       return 1;
  96   }
  97
  98   /* Matched! */
  99   return 0;
 100 }
 101
 102 static uintptr_t signextIfWin64(void *V) {
 103 #ifdef _WIN64
 104   return (uintptr_t)(int32_t)(uintptr_t)V;
 105 #else
 106   return (uintptr_t)V;
 107 #endif
 108 }
 109
 110 // Skip names section, vtable profile data section and vtable names section
 111 // for runtime profile merge. To merge runtime addresses from multiple
 112 // profiles collected from the same instrumented binary, the binary should be
 113 // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR
 114 // disabled). In this set-up these three sections remain unchanged.
 115 static uint64_t
 116 getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) {
 117   const uint64_t VTableSectionSize =
 118       Header->NumVTables * sizeof(VTableProfData);
 119   const uint64_t PaddingBytesAfterVTableSection =
 120       __llvm_profile_get_num_padding_bytes(VTableSectionSize);
 121   const uint64_t VNamesSize = Header->VNamesSize;
 122   const uint64_t PaddingBytesAfterVNamesSize =
 123       __llvm_profile_get_num_padding_bytes(VNamesSize);
 124   return Header->NamesSize +
 125          __llvm_profile_get_num_padding_bytes(Header->NamesSize) +
 126          VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize +
 127          PaddingBytesAfterVNamesSize;
 128 }
 129
 130 COMPILER_RT_VISIBILITY
 131 int __llvm_profile_merge_from_buffer(const char *ProfileData,
 132                                      uint64_t ProfileSize) {
 133   if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
 134     PROF_ERR("%s\n",
 135              "Temporal profiles do not support profile merging at runtime. "
 136              "Instead, merge raw profiles using the llvm-profdata tool.");
 137     return 1;
 138   }
 139
 140   __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
 141   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
 142   char *SrcCountersStart, *DstCounter;
 143   const char *SrcCountersEnd, *SrcCounter;
 144   const char *SrcBitmapStart;
 145   const char *SrcNameStart;
 146   const char *SrcValueProfDataStart, *SrcValueProfData;
 147   uintptr_t CountersDelta = Header->CountersDelta;
 148   uintptr_t BitmapDelta = Header->BitmapDelta;
 149
 150   SrcDataStart =
 151       (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
 152                               Header->BinaryIdsSize);
 153   SrcDataEnd = SrcDataStart + Header->NumData;
 154   SrcCountersStart = (char *)SrcDataEnd;
 155   SrcCountersEnd = SrcCountersStart +
 156                    Header->NumCounters * __llvm_profile_counter_entry_size();
 157   SrcBitmapStart = SrcCountersEnd + __llvm_profile_get_num_padding_bytes(
 158                                         SrcCountersEnd - SrcCountersStart);
 159   SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
 160   SrcValueProfDataStart =
 161       SrcNameStart + getDistanceFromCounterToValueProf(Header);
 162   if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
 163     return 1;
 164
 165   // Merge counters by iterating the entire counter section when data section is
 166   // empty due to correlation.
 167   if (Header->NumData == 0) {
 168     for (SrcCounter = SrcCountersStart,
 169         DstCounter = __llvm_profile_begin_counters();
 170          SrcCounter < SrcCountersEnd;) {
 171       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
 172         *DstCounter &= *SrcCounter;
 173       } else {
 174         *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
 175       }
 176       SrcCounter += __llvm_profile_counter_entry_size();
 177       DstCounter += __llvm_profile_counter_entry_size();
 178     }
 179     return 0;
 180   }
 181
 182   for (SrcData = SrcDataStart,
 183       DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
 184       SrcValueProfData = SrcValueProfDataStart;
 185        SrcData < SrcDataEnd; ++SrcData, ++DstData) {
 186     // For the in-memory destination, CounterPtr is the distance from the start
 187     // address of the data to the start address of the counter. On WIN64,
 188     // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
 189     // extend CounterPtr to get the original value.
 190     char *DstCounters =
 191         (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
 192     char *DstBitmap =
 193         (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
 194     unsigned NVK = 0;
 195
 196     // SrcData is a serialized representation of the memory image. We need to
 197     // compute the in-buffer counter offset from the in-memory address distance.
 198     // The initial CountersDelta is the in-memory address difference
 199     // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
 200     // CountersDelta computes the offset into the in-buffer counter section.
 201     //
 202     // On WIN64, CountersDelta is truncated as well, so no need for signext.
 203     char *SrcCounters =
 204         SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
 205     // CountersDelta needs to be decreased as we advance to the next data
 206     // record.
 207     CountersDelta -= sizeof(*SrcData);
 208     unsigned NC = SrcData->NumCounters;
 209     if (NC == 0)
 210       return 1;
 211     if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
 212         (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
 213       return 1;
 214     for (unsigned I = 0; I < NC; I++) {
 215       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
 216         // A value of zero signifies the function is covered.
 217         DstCounters[I] &= SrcCounters[I];
 218       } else {
 219         ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
 220       }
 221     }
 222
 223     const char *SrcBitmap =
 224         SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
 225     // BitmapDelta also needs to be decreased as we advance to the next data
 226     // record.
 227     BitmapDelta -= sizeof(*SrcData);
 228     unsigned NB = SrcData->NumBitmapBytes;
 229     // NumBitmapBytes may legitimately be 0. Just keep going.
 230     if (NB != 0) {
 231       if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
 232         return 1;
 233       // Merge Src and Dst Bitmap bytes by simply ORing them together.
 234       for (unsigned I = 0; I < NB; I++)
 235         DstBitmap[I] |= SrcBitmap[I];
 236     }
 237
 238     /* Now merge value profile data. */
 239     if (!VPMergeHook)
 240       continue;
 241
 242     for (unsigned I = 0; I <= IPVK_Last; I++)
 243       NVK += (SrcData->NumValueSites[I] != 0);
 244
 245     if (!NVK)
 246       continue;
 247
 248     if (SrcValueProfData >= ProfileData + ProfileSize)
 249       return 1;
 250     VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
 251     SrcValueProfData =
 252         SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
 253   }
 254
 255   return 0;
 256 }
 257
 258 #ifdef __GNUC__
 259 #pragma GCC diagnostic pop
 260 #elif defined(__clang__)
 261 #pragma clang diagnostic pop
 262 #endif