compiler-rt/lib/profile/InstrProfilingBuffer.c

   1 /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 \*===----------------------------------------------------------------------===*/
   8
   9 // Note: This is linked into the Darwin kernel, and must remain compatible
  10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
  11
  12 #include "InstrProfiling.h"
  13 #include "InstrProfilingInternal.h"
  14 #include "InstrProfilingPort.h"
  15
  16 /* When continuous mode is enabled (%c), this parameter is set to 1.
  17  *
  18  * This parameter is defined here in InstrProfilingBuffer.o, instead of in
  19  * InstrProfilingFile.o, to sequester all libc-dependent code in
  20  * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
  21  * layering is violated. */
  22 static int ContinuouslySyncProfile = 0;
  23
  24 /* The system page size. Only valid when non-zero. If 0, the page size is
  25  * unavailable. */
  26 static unsigned PageSize = 0;
  27
  28 COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
  29   return ContinuouslySyncProfile && PageSize;
  30 }
  31
  32 COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
  33   ContinuouslySyncProfile = 1;
  34 }
  35
  36 COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
  37   PageSize = PS;
  38 }
  39
  40 COMPILER_RT_VISIBILITY
  41 uint64_t __llvm_profile_get_size_for_buffer(void) {
  42   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
  43   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
  44   const char *CountersBegin = __llvm_profile_begin_counters();
  45   const char *CountersEnd = __llvm_profile_end_counters();
  46   const char *BitmapBegin = __llvm_profile_begin_bitmap();
  47   const char *BitmapEnd = __llvm_profile_end_bitmap();
  48   const char *NamesBegin = __llvm_profile_begin_names();
  49   const char *NamesEnd = __llvm_profile_end_names();
  50
  51   return __llvm_profile_get_size_for_buffer_internal(
  52       DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,
  53       NamesBegin, NamesEnd);
  54 }
  55
  56 COMPILER_RT_VISIBILITY
  57 uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
  58                                      const __llvm_profile_data *End) {
  59   if (hasCorrelation())
  60     return 0;
  61   intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
  62   return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
  63          sizeof(__llvm_profile_data);
  64 }
  65
  66 COMPILER_RT_VISIBILITY
  67 uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
  68                                       const __llvm_profile_data *End) {
  69   if (hasCorrelation())
  70     return 0;
  71   return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
  72 }
  73
  74 COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
  75   if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
  76     return sizeof(uint8_t);
  77   return sizeof(uint64_t);
  78 }
  79
  80 COMPILER_RT_VISIBILITY
  81 uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
  82   intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
  83   return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
  84          __llvm_profile_counter_entry_size();
  85 }
  86
  87 COMPILER_RT_VISIBILITY
  88 uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
  89   return __llvm_profile_get_num_counters(Begin, End) *
  90          __llvm_profile_counter_entry_size();
  91 }
  92
  93 COMPILER_RT_VISIBILITY
  94 uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
  95                                              const char *End) {
  96   return (End - Begin);
  97 }
  98
  99 COMPILER_RT_VISIBILITY
 100 uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
 101   if (hasCorrelation())
 102     return 0;
 103   return End - Begin;
 104 }
 105
 106 /// Calculate the number of padding bytes needed to add to \p Offset in order
 107 /// for (\p Offset + Padding) to be page-aligned.
 108 static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
 109   uint64_t OffsetModPage = Offset % PageSize;
 110   if (OffsetModPage > 0)
 111     return PageSize - OffsetModPage;
 112   return 0;
 113 }
 114
 115 static int needsCounterPadding(void) {
 116 #if defined(__APPLE__)
 117   return __llvm_profile_is_continuous_mode_enabled();
 118 #else
 119   return 0;
 120 #endif
 121 }
 122
 123 COMPILER_RT_VISIBILITY
 124 void __llvm_profile_get_padding_sizes_for_counters(
 125     uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
 126     uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters,
 127     uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes,
 128     uint64_t *PaddingBytesAfterNames) {
 129   if (!needsCounterPadding()) {
 130     *PaddingBytesBeforeCounters = 0;
 131     *PaddingBytesAfterCounters =
 132         __llvm_profile_get_num_padding_bytes(CountersSize);
 133     *PaddingBytesAfterBitmapBytes =
 134         __llvm_profile_get_num_padding_bytes(NumBitmapBytes);
 135     *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
 136     return;
 137   }
 138
 139   // In continuous mode, the file offsets for headers and for the start of
 140   // counter sections need to be page-aligned.
 141   *PaddingBytesBeforeCounters =
 142       calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
 143   *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
 144   *PaddingBytesAfterBitmapBytes =
 145       calculateBytesNeededToPageAlign(NumBitmapBytes);
 146   *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
 147 }
 148
 149 COMPILER_RT_VISIBILITY
 150 uint64_t __llvm_profile_get_size_for_buffer_internal(
 151     const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
 152     const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
 153     const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) {
 154   /* Match logic in __llvm_profile_write_buffer(). */
 155   const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
 156   uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
 157   uint64_t CountersSize =
 158       __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
 159   const uint64_t NumBitmapBytes =
 160       __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
 161
 162   /* Determine how much padding is needed before/after the counters and after
 163    * the names. */
 164   uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
 165       PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
 166   __llvm_profile_get_padding_sizes_for_counters(
 167       DataSize, CountersSize, NumBitmapBytes, NamesSize,
 168       &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
 169       &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames);
 170
 171   return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
 172          DataSize + PaddingBytesBeforeCounters + CountersSize +
 173          PaddingBytesAfterCounters + NumBitmapBytes +
 174          PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames;
 175 }
 176
 177 COMPILER_RT_VISIBILITY
 178 void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {
 179   BufferWriter->Write = lprofBufferWriter;
 180   BufferWriter->WriterCtx = Buffer;
 181 }
 182
 183 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
 184   ProfDataWriter BufferWriter;
 185   initBufferWriter(&BufferWriter, Buffer);
 186   return lprofWriteData(&BufferWriter, 0, 0);
 187 }
 188
 189 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
 190     char *Buffer, const __llvm_profile_data *DataBegin,
 191     const __llvm_profile_data *DataEnd, const char *CountersBegin,
 192     const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
 193     const char *NamesBegin, const char *NamesEnd) {
 194   ProfDataWriter BufferWriter;
 195   initBufferWriter(&BufferWriter, Buffer);
 196   return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
 197                             CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin,
 198                             NamesEnd, 0);
 199 }