compiler-rt/lib/profile/InstrProfilingMerge.c

   1 /*===- InstrProfilingMerge.c - Profile in-process Merging  ---------------===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 |*===----------------------------------------------------------------------===*
   8 |* This file defines the API needed for in-process merging of profile data
   9 |* stored in memory buffer.
  10 \*===---------------------------------------------------------------------===*/
  11
  12 #include "InstrProfiling.h"
  13 #include "InstrProfilingInternal.h"
  14 #include "InstrProfilingUtil.h"
  15
  16 #define INSTR_PROF_VALUE_PROF_DATA
  17 #include "profile/InstrProfData.inc"
  18
  19 COMPILER_RT_VISIBILITY
  20 void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
  21
  22 COMPILER_RT_VISIBILITY
  23 uint64_t lprofGetLoadModuleSignature(void) {
  24   /* A very fast way to compute a module signature.  */
  25   uint64_t Version = __llvm_profile_get_version();
  26   uint64_t NumCounters = __llvm_profile_get_num_counters(
  27       __llvm_profile_begin_counters(), __llvm_profile_end_counters());
  28   uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
  29                                                  __llvm_profile_end_data());
  30   uint64_t NamesSize =
  31       (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
  32   uint64_t NumVnodes =
  33       (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
  34   const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
  35
  36   return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
  37          (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
  38          __llvm_profile_get_magic();
  39 }
  40
  41 #ifdef __GNUC__
  42 #pragma GCC diagnostic push
  43 #pragma GCC diagnostic ignored "-Wcast-qual"
  44 #endif
  45
  46 /* Returns 1 if profile is not structurally compatible.  */
  47 COMPILER_RT_VISIBILITY
  48 int __llvm_profile_check_compatibility(const char *ProfileData,
  49                                        uint64_t ProfileSize) {
  50   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
  51   __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
  52   SrcDataStart =
  53       (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
  54                               Header->BinaryIdsSize);
  55   SrcDataEnd = SrcDataStart + Header->NumData;
  56
  57   if (ProfileSize < sizeof(__llvm_profile_header))
  58     return 1;
  59
  60   /* Check the header first.  */
  61   if (Header->Magic != __llvm_profile_get_magic() ||
  62       Header->Version != __llvm_profile_get_version() ||
  63       Header->NumData !=
  64           __llvm_profile_get_num_data(__llvm_profile_begin_data(),
  65                                       __llvm_profile_end_data()) ||
  66       Header->NumCounters !=
  67           __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
  68                                           __llvm_profile_end_counters()) ||
  69       Header->NumBitmapBytes !=
  70           __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
  71                                               __llvm_profile_end_bitmap()) ||
  72       Header->NamesSize !=
  73           __llvm_profile_get_name_size(__llvm_profile_begin_names(),
  74                                        __llvm_profile_end_names()) ||
  75       Header->ValueKindLast != IPVK_Last)
  76     return 1;
  77
  78   if (ProfileSize <
  79       sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
  80           Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
  81           Header->NumCounters * __llvm_profile_counter_entry_size() +
  82           Header->NumBitmapBytes)
  83     return 1;
  84
  85   for (SrcData = SrcDataStart,
  86        DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
  87        SrcData < SrcDataEnd; ++SrcData, ++DstData) {
  88     if (SrcData->NameRef != DstData->NameRef ||
  89         SrcData->FuncHash != DstData->FuncHash ||
  90         SrcData->NumCounters != DstData->NumCounters ||
  91         SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
  92       return 1;
  93   }
  94
  95   /* Matched! */
  96   return 0;
  97 }
  98
  99 static uintptr_t signextIfWin64(void *V) {
 100 #ifdef _WIN64
 101   return (uintptr_t)(int32_t)(uintptr_t)V;
 102 #else
 103   return (uintptr_t)V;
 104 #endif
 105 }
 106
 107 COMPILER_RT_VISIBILITY
 108 int __llvm_profile_merge_from_buffer(const char *ProfileData,
 109                                      uint64_t ProfileSize) {
 110   if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
 111     PROF_ERR("%s\n",
 112              "Temporal profiles do not support profile merging at runtime. "
 113              "Instead, merge raw profiles using the llvm-profdata tool.");
 114     return 1;
 115   }
 116
 117   __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
 118   __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
 119   char *SrcCountersStart, *DstCounter;
 120   const char *SrcCountersEnd, *SrcCounter;
 121   const char *SrcBitmapStart;
 122   const char *SrcNameStart;
 123   const char *SrcValueProfDataStart, *SrcValueProfData;
 124   uintptr_t CountersDelta = Header->CountersDelta;
 125   uintptr_t BitmapDelta = Header->BitmapDelta;
 126
 127   SrcDataStart =
 128       (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
 129                               Header->BinaryIdsSize);
 130   SrcDataEnd = SrcDataStart + Header->NumData;
 131   SrcCountersStart = (char *)SrcDataEnd;
 132   SrcCountersEnd = SrcCountersStart +
 133                    Header->NumCounters * __llvm_profile_counter_entry_size();
 134   SrcBitmapStart = SrcCountersEnd;
 135   SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
 136   SrcValueProfDataStart =
 137       SrcNameStart + Header->NamesSize +
 138       __llvm_profile_get_num_padding_bytes(Header->NamesSize);
 139   if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
 140     return 1;
 141
 142   // Merge counters by iterating the entire counter section when correlation is
 143   // enabled.
 144   if (hasCorrelation()) {
 145     for (SrcCounter = SrcCountersStart,
 146         DstCounter = __llvm_profile_begin_counters();
 147          SrcCounter < SrcCountersEnd;) {
 148       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
 149         *DstCounter &= *SrcCounter;
 150       } else {
 151         *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
 152       }
 153       SrcCounter += __llvm_profile_counter_entry_size();
 154       DstCounter += __llvm_profile_counter_entry_size();
 155     }
 156     return 0;
 157   }
 158
 159   for (SrcData = SrcDataStart,
 160       DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
 161       SrcValueProfData = SrcValueProfDataStart;
 162        SrcData < SrcDataEnd; ++SrcData, ++DstData) {
 163     // For the in-memory destination, CounterPtr is the distance from the start
 164     // address of the data to the start address of the counter. On WIN64,
 165     // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
 166     // extend CounterPtr to get the original value.
 167     char *DstCounters =
 168         (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
 169     char *DstBitmap =
 170         (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
 171     unsigned NVK = 0;
 172
 173     // SrcData is a serialized representation of the memory image. We need to
 174     // compute the in-buffer counter offset from the in-memory address distance.
 175     // The initial CountersDelta is the in-memory address difference
 176     // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
 177     // CountersDelta computes the offset into the in-buffer counter section.
 178     //
 179     // On WIN64, CountersDelta is truncated as well, so no need for signext.
 180     char *SrcCounters =
 181         SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
 182     // CountersDelta needs to be decreased as we advance to the next data
 183     // record.
 184     CountersDelta -= sizeof(*SrcData);
 185     unsigned NC = SrcData->NumCounters;
 186     if (NC == 0)
 187       return 1;
 188     if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
 189         (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
 190       return 1;
 191     for (unsigned I = 0; I < NC; I++) {
 192       if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
 193         // A value of zero signifies the function is covered.
 194         DstCounters[I] &= SrcCounters[I];
 195       } else {
 196         ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
 197       }
 198     }
 199
 200     const char *SrcBitmap =
 201         SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
 202     // BitmapDelta also needs to be decreased as we advance to the next data
 203     // record.
 204     BitmapDelta -= sizeof(*SrcData);
 205     unsigned NB = SrcData->NumBitmapBytes;
 206     // NumBitmapBytes may legitimately be 0. Just keep going.
 207     if (NB != 0) {
 208       if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
 209         return 1;
 210       // Merge Src and Dst Bitmap bytes by simply ORing them together.
 211       for (unsigned I = 0; I < NB; I++)
 212         DstBitmap[I] |= SrcBitmap[I];
 213     }
 214
 215     /* Now merge value profile data. */
 216     if (!VPMergeHook)
 217       continue;
 218
 219     for (unsigned I = 0; I <= IPVK_Last; I++)
 220       NVK += (SrcData->NumValueSites[I] != 0);
 221
 222     if (!NVK)
 223       continue;
 224
 225     if (SrcValueProfData >= ProfileData + ProfileSize)
 226       return 1;
 227     VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
 228     SrcValueProfData =
 229         SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
 230   }
 231
 232   return 0;
 233 }
 234
 235 #ifdef __GNUC__
 236 #pragma GCC diagnostic pop
 237 #endif