compiler-rt/lib/profile/InstrProfilingBuffer.c

   1 /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 \*===----------------------------------------------------------------------===*/
   8
   9 // Note: This is linked into the Darwin kernel, and must remain compatible
  10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
  11
  12 #include "InstrProfiling.h"
  13 #include "InstrProfilingInternal.h"
  14 #include "InstrProfilingPort.h"
  15
  16 /* When continuous mode is enabled (%c), this parameter is set to 1.
  17  *
  18  * This parameter is defined here in InstrProfilingBuffer.o, instead of in
  19  * InstrProfilingFile.o, to sequester all libc-dependent code in
  20  * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
  21  * layering is violated. */
  22 static int ContinuouslySyncProfile = 0;
  23
  24 /* The system page size. Only valid when non-zero. If 0, the page size is
  25  * unavailable. */
  26 static unsigned PageSize = 0;
  27
  28 COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
  29   return ContinuouslySyncProfile && PageSize;
  30 }
  31
  32 COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
  33   ContinuouslySyncProfile = 1;
  34 }
  35
  36 COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) {
  37   ContinuouslySyncProfile = 0;
  38 }
  39
  40 COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
  41   PageSize = PS;
  42 }
  43
  44 COMPILER_RT_VISIBILITY
  45 uint64_t __llvm_profile_get_size_for_buffer(void) {
  46   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
  47   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
  48   const char *CountersBegin = __llvm_profile_begin_counters();
  49   const char *CountersEnd = __llvm_profile_end_counters();
  50   const char *BitmapBegin = __llvm_profile_begin_bitmap();
  51   const char *BitmapEnd = __llvm_profile_end_bitmap();
  52   const char *NamesBegin = __llvm_profile_begin_names();
  53   const char *NamesEnd = __llvm_profile_end_names();
  54   const VTableProfData *VTableBegin = __llvm_profile_begin_vtables();
  55   const VTableProfData *VTableEnd = __llvm_profile_end_vtables();
  56   const char *VNamesBegin = __llvm_profile_begin_vtabnames();
  57   const char *VNamesEnd = __llvm_profile_end_vtabnames();
  58
  59   return __llvm_profile_get_size_for_buffer_internal(
  60       DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,
  61       NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd);
  62 }
  63
  64 // NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open
  65 // interval [Begin, End). Namely, `End` is one-byte past the end of the array.
  66 COMPILER_RT_VISIBILITY
  67 uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
  68                                      const __llvm_profile_data *End) {
  69   intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
  70   return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
  71          sizeof(__llvm_profile_data);
  72 }
  73
  74 COMPILER_RT_VISIBILITY
  75 uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
  76                                       const __llvm_profile_data *End) {
  77   return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
  78 }
  79
  80 // Counts the number of `VTableProfData` elements within the range of [Begin,
  81 // End). Caller should guarantee that End points to one byte past the inclusive
  82 // range.
  83 // FIXME: Add a compiler-rt test to make sure the number of vtables in the
  84 // raw profile is the same as the number of vtable elements in the instrumented
  85 // binary.
  86 COMPILER_RT_VISIBILITY
  87 uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin,
  88                                        const VTableProfData *End) {
  89   // Convert pointers to intptr_t to use integer arithmetic.
  90   intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin;
  91   return (EndI - BeginI) / sizeof(VTableProfData);
  92 }
  93
  94 COMPILER_RT_VISIBILITY
  95 uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin,
  96                                                 const VTableProfData *End) {
  97   return (intptr_t)(End) - (intptr_t)(Begin);
  98 }
  99
 100 COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
 101   if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
 102     return sizeof(uint8_t);
 103   return sizeof(uint64_t);
 104 }
 105
 106 COMPILER_RT_VISIBILITY
 107 uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
 108   intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
 109   return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
 110          __llvm_profile_counter_entry_size();
 111 }
 112
 113 COMPILER_RT_VISIBILITY
 114 uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
 115   return __llvm_profile_get_num_counters(Begin, End) *
 116          __llvm_profile_counter_entry_size();
 117 }
 118
 119 COMPILER_RT_VISIBILITY
 120 uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
 121                                              const char *End) {
 122   return (End - Begin);
 123 }
 124
 125 COMPILER_RT_VISIBILITY
 126 uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
 127   return End - Begin;
 128 }
 129
 130 /// Calculate the number of padding bytes needed to add to \p Offset in order
 131 /// for (\p Offset + Padding) to be page-aligned.
 132 static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
 133   uint64_t OffsetModPage = Offset % PageSize;
 134   if (OffsetModPage > 0)
 135     return PageSize - OffsetModPage;
 136   return 0;
 137 }
 138
 139 static int needsCounterPadding(void) {
 140 #if defined(__APPLE__)
 141   return __llvm_profile_is_continuous_mode_enabled();
 142 #else
 143   return 0;
 144 #endif
 145 }
 146
 147 COMPILER_RT_VISIBILITY
 148 int __llvm_profile_get_padding_sizes_for_counters(
 149     uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
 150     uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize,
 151     uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters,
 152     uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames,
 153     uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) {
 154   // Counter padding is needed only if continuous mode is enabled.
 155   if (!needsCounterPadding()) {
 156     *PaddingBytesBeforeCounters = 0;
 157     *PaddingBytesAfterCounters =
 158         __llvm_profile_get_num_padding_bytes(CountersSize);
 159     *PaddingBytesAfterBitmapBytes =
 160         __llvm_profile_get_num_padding_bytes(NumBitmapBytes);
 161     *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
 162     if (PaddingBytesAfterVTable != NULL)
 163       *PaddingBytesAfterVTable =
 164           __llvm_profile_get_num_padding_bytes(VTableSize);
 165     if (PaddingBytesAfterVName != NULL)
 166       *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize);
 167     return 0;
 168   }
 169
 170   // Value profiling not supported in continuous mode at profile-write time.
 171   // Return -1 to alert the incompatibility.
 172   if (VTableSize != 0 || VNameSize != 0)
 173     return -1;
 174
 175   // In continuous mode, the file offsets for headers and for the start of
 176   // counter sections need to be page-aligned.
 177   *PaddingBytesBeforeCounters =
 178       calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
 179   *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
 180   *PaddingBytesAfterBitmapBytes =
 181       calculateBytesNeededToPageAlign(NumBitmapBytes);
 182   *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
 183   // Set these two variables to zero to avoid uninitialized variables
 184   // even if VTableSize and VNameSize are known to be zero.
 185   if (PaddingBytesAfterVTable != NULL)
 186     *PaddingBytesAfterVTable = 0;
 187   if (PaddingBytesAfterVName != NULL)
 188     *PaddingBytesAfterVName = 0;
 189   return 0;
 190 }
 191
 192 COMPILER_RT_VISIBILITY
 193 uint64_t __llvm_profile_get_size_for_buffer_internal(
 194     const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
 195     const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
 196     const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd,
 197     const VTableProfData *VTableBegin, const VTableProfData *VTableEnd,
 198     const char *VNamesBegin, const char *VNamesEnd) {
 199   /* Match logic in __llvm_profile_write_buffer(). */
 200   const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
 201   uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
 202   uint64_t CountersSize =
 203       __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
 204   const uint64_t NumBitmapBytes =
 205       __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
 206   const uint64_t VTableSize =
 207       __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd);
 208   const uint64_t VNameSize =
 209       __llvm_profile_get_name_size(VNamesBegin, VNamesEnd);
 210
 211   /* Determine how much padding is needed before/after the counters and after
 212    * the names. */
 213   uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
 214       PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes,
 215       PaddingBytesAfterVTable, PaddingBytesAfterVNames;
 216   __llvm_profile_get_padding_sizes_for_counters(
 217       DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */,
 218       0 /* VNameSize */, &PaddingBytesBeforeCounters,
 219       &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes,
 220       &PaddingBytesAfterNames, &PaddingBytesAfterVTable,
 221       &PaddingBytesAfterVNames);
 222
 223   return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
 224          DataSize + PaddingBytesBeforeCounters + CountersSize +
 225          PaddingBytesAfterCounters + NumBitmapBytes +
 226          PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames +
 227          VTableSize + PaddingBytesAfterVTable + VNameSize +
 228          PaddingBytesAfterVNames;
 229 }
 230
 231 COMPILER_RT_VISIBILITY
 232 void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {
 233   BufferWriter->Write = lprofBufferWriter;
 234   BufferWriter->WriterCtx = Buffer;
 235 }
 236
 237 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
 238   ProfDataWriter BufferWriter;
 239   initBufferWriter(&BufferWriter, Buffer);
 240   return lprofWriteData(&BufferWriter, 0, 0);
 241 }
 242
 243 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
 244     char *Buffer, const __llvm_profile_data *DataBegin,
 245     const __llvm_profile_data *DataEnd, const char *CountersBegin,
 246     const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
 247     const char *NamesBegin, const char *NamesEnd) {
 248   ProfDataWriter BufferWriter;
 249   initBufferWriter(&BufferWriter, Buffer);
 250   // Set virtual table arguments to NULL since they are not supported yet.
 251   return lprofWriteDataImpl(
 252       &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd,
 253       BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd,
 254       /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL,
 255       /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0);
 256 }