compiler-rt/lib/profile/InstrProfilingWriter.c

   1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
   2 |*
   3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 |* See https://llvm.org/LICENSE.txt for license information.
   5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 |*
   7 \*===----------------------------------------------------------------------===*/
   8
   9 // Note: This is linked into the Darwin kernel, and must remain compatible
  10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
  11
  12 #ifdef _MSC_VER
  13 /* For _alloca */
  14 #include <malloc.h>
  15 #endif
  16 #include <string.h>
  17
  18 #include "InstrProfiling.h"
  19 #include "InstrProfilingInternal.h"
  20 #include "InstrProfilingPort.h"
  21
  22 #define INSTR_PROF_VALUE_PROF_DATA
  23 #include "profile/InstrProfData.inc"
  24
  25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
  26 static ProfBufferIO TheBufferIO;
  27 #define VP_BUFFER_SIZE 8 * 1024
  28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
  29 static InstrProfValueData VPDataArray[16];
  30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
  31
  32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
  33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
  34
  35 /* The buffer writer is responsible in keeping writer state
  36  * across the call.
  37  */
  38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
  39                                                   ProfDataIOVec *IOVecs,
  40                                                   uint32_t NumIOVecs) {
  41   uint32_t I;
  42   char **Buffer = (char **)&This->WriterCtx;
  43   for (I = 0; I < NumIOVecs; I++) {
  44     size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
  45     if (IOVecs[I].Data)
  46       memcpy(*Buffer, IOVecs[I].Data, Length);
  47     else if (IOVecs[I].UseZeroPadding) {
  48       /* Allocating the buffer should zero fill. */
  49     }
  50     *Buffer += Length;
  51   }
  52   return 0;
  53 }
  54
  55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
  56                              uint8_t *Buffer, uint32_t BufferSz) {
  57   BufferIO->FileWriter = FileWriter;
  58   BufferIO->OwnFileWriter = 0;
  59   BufferIO->BufferStart = Buffer;
  60   BufferIO->BufferSz = BufferSz;
  61   BufferIO->CurOffset = 0;
  62 }
  63
  64 COMPILER_RT_VISIBILITY ProfBufferIO *
  65 lprofCreateBufferIO(ProfDataWriter *FileWriter) {
  66   uint8_t *Buffer = DynamicBufferIOBuffer;
  67   uint32_t BufferSize = VPBufferSize;
  68   if (!Buffer) {
  69     Buffer = &BufferIOBuffer[0];
  70     BufferSize = sizeof(BufferIOBuffer);
  71   }
  72   llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
  73   return &TheBufferIO;
  74 }
  75
  76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
  77   if (BufferIO->OwnFileWriter)
  78     FreeHook(BufferIO->FileWriter);
  79   if (DynamicBufferIOBuffer) {
  80     FreeHook(DynamicBufferIOBuffer);
  81     DynamicBufferIOBuffer = 0;
  82     VPBufferSize = 0;
  83   }
  84 }
  85
  86 COMPILER_RT_VISIBILITY int
  87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
  88   /* Buffer is not large enough, it is time to flush.  */
  89   if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
  90     if (lprofBufferIOFlush(BufferIO) != 0)
  91       return -1;
  92   }
  93   /* Special case, bypass the buffer completely. */
  94   ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
  95   if (Size > BufferIO->BufferSz) {
  96     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
  97       return -1;
  98   } else {
  99     /* Write the data to buffer */
 100     uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
 101     ProfDataWriter BufferWriter;
 102     initBufferWriter(&BufferWriter, (char *)Buffer);
 103     lprofBufferWriter(&BufferWriter, IO, 1);
 104     BufferIO->CurOffset =
 105         (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
 106   }
 107   return 0;
 108 }
 109
 110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
 111   if (BufferIO->CurOffset) {
 112     ProfDataIOVec IO[] = {
 113         {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
 114     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
 115       return -1;
 116     BufferIO->CurOffset = 0;
 117   }
 118   return 0;
 119 }
 120
 121 /* Write out value profile data for function specified with \c Data.
 122  * The implementation does not use the method \c serializeValueProfData
 123  * which depends on dynamic memory allocation. In this implementation,
 124  * value profile data is written out to \c BufferIO piecemeal.
 125  */
 126 static int writeOneValueProfData(ProfBufferIO *BufferIO,
 127                                  VPDataReaderType *VPDataReader,
 128                                  const __llvm_profile_data *Data) {
 129   unsigned I, NumValueKinds = 0;
 130   ValueProfData VPHeader;
 131   uint8_t *SiteCountArray[IPVK_Last + 1];
 132
 133   for (I = 0; I <= IPVK_Last; I++) {
 134     if (!Data->NumValueSites[I])
 135       SiteCountArray[I] = 0;
 136     else {
 137       uint32_t Sz =
 138           VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
 139           offsetof(ValueProfRecord, SiteCountArray);
 140       /* Only use alloca for this small byte array to avoid excessive
 141        * stack growth.  */
 142       SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
 143       memset(SiteCountArray[I], 0, Sz);
 144     }
 145   }
 146
 147   /* If NumValueKinds returned is 0, there is nothing to write, report
 148      success and return. This should match the raw profile reader's behavior. */
 149   if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
 150     return 0;
 151
 152   /* First write the header structure. */
 153   VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
 154   VPHeader.NumValueKinds = NumValueKinds;
 155   if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
 156                          sizeof(ValueProfData)))
 157     return -1;
 158
 159   /* Make sure nothing else needs to be written before value profile
 160    * records. */
 161   if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
 162       (void *)(&VPHeader + 1))
 163     return -1;
 164
 165   /* Write out the value profile record for each value kind
 166    * one by one. */
 167   for (I = 0; I <= IPVK_Last; I++) {
 168     uint32_t J;
 169     ValueProfRecord RecordHeader;
 170     /* The size of the value prof record header without counting the
 171      * site count array .*/
 172     uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
 173     uint32_t SiteCountArraySize;
 174
 175     if (!Data->NumValueSites[I])
 176       continue;
 177
 178     /* Write out the record header.  */
 179     RecordHeader.Kind = I;
 180     RecordHeader.NumValueSites = Data->NumValueSites[I];
 181     if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
 182                            RecordHeaderSize))
 183       return -1;
 184
 185     /* Write out the site value count array including padding space. */
 186     SiteCountArraySize =
 187         VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
 188         RecordHeaderSize;
 189     if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
 190       return -1;
 191
 192     /* Write out the value profile data for each value site.  */
 193     for (J = 0; J < Data->NumValueSites[I]; J++) {
 194       uint32_t NRead, NRemain;
 195       ValueProfNode *NextStartNode = 0;
 196       NRemain = VPDataReader->GetNumValueDataForSite(I, J);
 197       if (!NRemain)
 198         continue;
 199       /* Read and write out value data in small chunks till it is done. */
 200       do {
 201         NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
 202         NextStartNode =
 203             VPDataReader->GetValueData(I, /* ValueKind */
 204                                        J, /* Site */
 205                                        &VPDataArray[0], NextStartNode, NRead);
 206         if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
 207                                NRead * sizeof(InstrProfValueData)))
 208           return -1;
 209         NRemain -= NRead;
 210       } while (NRemain != 0);
 211     }
 212   }
 213   /* All done report success.  */
 214   return 0;
 215 }
 216
 217 static int writeValueProfData(ProfDataWriter *Writer,
 218                               VPDataReaderType *VPDataReader,
 219                               const __llvm_profile_data *DataBegin,
 220                               const __llvm_profile_data *DataEnd) {
 221   ProfBufferIO *BufferIO;
 222   const __llvm_profile_data *DI = 0;
 223
 224   if (!VPDataReader)
 225     return 0;
 226
 227   BufferIO = lprofCreateBufferIO(Writer);
 228
 229   for (DI = DataBegin; DI < DataEnd; DI++) {
 230     if (writeOneValueProfData(BufferIO, VPDataReader, DI))
 231       return -1;
 232   }
 233
 234   if (lprofBufferIOFlush(BufferIO) != 0)
 235     return -1;
 236   lprofDeleteBufferIO(BufferIO);
 237
 238   return 0;
 239 }
 240
 241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
 242                                           VPDataReaderType *VPDataReader,
 243                                           int SkipNameDataWrite) {
 244   /* Match logic in __llvm_profile_write_buffer(). */
 245   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
 246   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
 247   const char *CountersBegin = __llvm_profile_begin_counters();
 248   const char *CountersEnd = __llvm_profile_end_counters();
 249   const char *BitmapBegin = __llvm_profile_begin_bitmap();
 250   const char *BitmapEnd = __llvm_profile_end_bitmap();
 251   const char *NamesBegin = __llvm_profile_begin_names();
 252   const char *NamesEnd = __llvm_profile_end_names();
 253   return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
 254                             CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
 255                             NamesBegin, NamesEnd, SkipNameDataWrite);
 256 }
 257
 258 COMPILER_RT_VISIBILITY int
 259 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
 260                    const __llvm_profile_data *DataEnd,
 261                    const char *CountersBegin, const char *CountersEnd,
 262                    const char *BitmapBegin, const char *BitmapEnd,
 263                    VPDataReaderType *VPDataReader, const char *NamesBegin,
 264                    const char *NamesEnd, int SkipNameDataWrite) {
 265   int ProfileCorrelation = hasCorrelation();
 266
 267   /* Calculate size of sections. */
 268   const uint64_t DataSectionSize =
 269       __llvm_profile_get_data_size(DataBegin, DataEnd);
 270   const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd);
 271   const uint64_t CountersSectionSize =
 272       __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
 273   const uint64_t NumCounters =
 274       __llvm_profile_get_num_counters(CountersBegin, CountersEnd);
 275   const uint64_t NumBitmapBytes =
 276       __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
 277   const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);
 278
 279   /* Create the header. */
 280   __llvm_profile_header Header;
 281
 282   /* Determine how much padding is needed before/after the counters and after
 283    * the names. */
 284   uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
 285       PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
 286   __llvm_profile_get_padding_sizes_for_counters(
 287       DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
 288       &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
 289       &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames);
 290
 291   {
 292 /* Initialize header structure.  */
 293 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
 294 #include "profile/InstrProfData.inc"
 295   }
 296
 297   /* On WIN64, label differences are truncated 32-bit values. Truncate
 298    * CountersDelta to match. */
 299 #ifdef _WIN64
 300   Header.CountersDelta = (uint32_t)Header.CountersDelta;
 301   Header.BitmapDelta = (uint32_t)Header.BitmapDelta;
 302 #endif
 303
 304   /* The data and names sections are omitted in lightweight mode. */
 305   if (ProfileCorrelation) {
 306     Header.CountersDelta = 0;
 307     Header.NamesDelta = 0;
 308   }
 309
 310   /* Write the profile header. */
 311   ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
 312   if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
 313     return -1;
 314
 315   /* Write the binary id lengths and data. */
 316   if (__llvm_write_binary_ids(Writer) == -1)
 317     return -1;
 318
 319   /* Write the profile data. */
 320   ProfDataIOVec IOVecData[] = {
 321       {ProfileCorrelation ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
 322        0},
 323       {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
 324       {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
 325       {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
 326       {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
 327       {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
 328       {(SkipNameDataWrite || ProfileCorrelation) ? NULL : NamesBegin,
 329        sizeof(uint8_t), NamesSize, 0},
 330       {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
 331   if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
 332     return -1;
 333
 334   /* Value profiling is not yet supported in continuous mode and profile
 335    * correlation mode. */
 336   if (__llvm_profile_is_continuous_mode_enabled() || ProfileCorrelation)
 337     return 0;
 338
 339   return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
 340 }
 341
 342 /*
 343  * Write binary id length and then its data, because binary id does not
 344  * have a fixed length.
 345  */
 346 COMPILER_RT_VISIBILITY
 347 int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
 348                           const uint8_t *BinaryIdData,
 349                           uint64_t BinaryIdPadding) {
 350   ProfDataIOVec BinaryIdIOVec[] = {
 351       {&BinaryIdLen, sizeof(uint64_t), 1, 0},
 352       {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
 353       {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
 354   };
 355   if (Writer->Write(Writer, BinaryIdIOVec,
 356                     sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
 357     return -1;
 358
 359   /* Successfully wrote binary id, report success. */
 360   return 0;
 361 }