compiler-rt/lib/memprof/memprof_rawprofile.cpp

   1 #include <stdint.h>
   2 #include <stdlib.h>
   3 #include <string.h>
   4
   5 #include "memprof_rawprofile.h"
   6 #include "profile/MemProfData.inc"
   7 #include "sanitizer_common/sanitizer_allocator_internal.h"
   8 #include "sanitizer_common/sanitizer_array_ref.h"
   9 #include "sanitizer_common/sanitizer_common.h"
  10 #include "sanitizer_common/sanitizer_linux.h"
  11 #include "sanitizer_common/sanitizer_procmaps.h"
  12 #include "sanitizer_common/sanitizer_stackdepot.h"
  13 #include "sanitizer_common/sanitizer_stackdepotbase.h"
  14 #include "sanitizer_common/sanitizer_stacktrace.h"
  15 #include "sanitizer_common/sanitizer_vector.h"
  16
  17 namespace __memprof {
  18 using ::__sanitizer::Vector;
  19 using ::llvm::memprof::MemInfoBlock;
  20 using SegmentEntry = ::llvm::memprof::SegmentEntry;
  21 using Header = ::llvm::memprof::Header;
  22
  23 namespace {
  24 template <class T> char *WriteBytes(const T &Pod, char *Buffer) {
  25   *(T *)Buffer = Pod;
  26   return Buffer + sizeof(T);
  27 }
  28
  29 void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,
  30                    void *Arg) {
  31   // No need to touch the MIB value here since we are only recording the key.
  32   auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);
  33   StackIds->PushBack(Key);
  34 }
  35 } // namespace
  36
  37 u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) {
  38   u64 NumSegmentsToRecord = 0;
  39   for (const auto &Module : Modules) {
  40     for (const auto &Segment : Module.ranges()) {
  41       if (Segment.executable)
  42         NumSegmentsToRecord++;
  43     }
  44   }
  45
  46   return sizeof(u64) // A header which stores the number of records.
  47          + sizeof(SegmentEntry) * NumSegmentsToRecord;
  48 }
  49
  50 // The segment section uses the following format:
  51 // ---------- Segment Info
  52 // Num Entries
  53 // ---------- Segment Entry
  54 // Start
  55 // End
  56 // Offset
  57 // UuidSize
  58 // Uuid 32B
  59 // ----------
  60 // ...
  61 void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules,
  62                                const u64 ExpectedNumBytes, char *&Buffer) {
  63   char *Ptr = Buffer;
  64   // Reserve space for the final count.
  65   Ptr += sizeof(u64);
  66
  67   u64 NumSegmentsRecorded = 0;
  68
  69   for (const auto &Module : Modules) {
  70     for (const auto &Segment : Module.ranges()) {
  71       if (Segment.executable) {
  72         SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address());
  73         CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE);
  74         Entry.BuildIdSize = Module.uuid_size();
  75         memcpy(Entry.BuildId, Module.uuid(), Module.uuid_size());
  76         memcpy(Ptr, &Entry, sizeof(SegmentEntry));
  77         Ptr += sizeof(SegmentEntry);
  78         NumSegmentsRecorded++;
  79       }
  80     }
  81   }
  82   // Store the number of segments we recorded in the space we reserved.
  83   *((u64 *)Buffer) = NumSegmentsRecorded;
  84   CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
  85         "Expected num bytes != actual bytes written");
  86 }
  87
  88 u64 StackSizeBytes(const Vector<u64> &StackIds) {
  89   u64 NumBytesToWrite = sizeof(u64);
  90
  91   const u64 NumIds = StackIds.Size();
  92   for (unsigned k = 0; k < NumIds; ++k) {
  93     const u64 Id = StackIds[k];
  94     // One entry for the id and then one more for the number of stack pcs.
  95     NumBytesToWrite += 2 * sizeof(u64);
  96     const StackTrace St = StackDepotGet(Id);
  97
  98     CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");
  99     for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
 100       NumBytesToWrite += sizeof(u64);
 101     }
 102   }
 103   return NumBytesToWrite;
 104 }
 105
 106 // The stack info section uses the following format:
 107 //
 108 // ---------- Stack Info
 109 // Num Entries
 110 // ---------- Stack Entry
 111 // Num Stacks
 112 // PC1
 113 // PC2
 114 // ...
 115 // ----------
 116 void SerializeStackToBuffer(const Vector<u64> &StackIds,
 117                             const u64 ExpectedNumBytes, char *&Buffer) {
 118   const u64 NumIds = StackIds.Size();
 119   char *Ptr = Buffer;
 120   Ptr = WriteBytes(static_cast<u64>(NumIds), Ptr);
 121
 122   for (unsigned k = 0; k < NumIds; ++k) {
 123     const u64 Id = StackIds[k];
 124     Ptr = WriteBytes(Id, Ptr);
 125     Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.
 126     u64 Count = 0;
 127     const StackTrace St = StackDepotGet(Id);
 128     for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {
 129       // PCs in stack traces are actually the return addresses, that is,
 130       // addresses of the next instructions after the call.
 131       uptr pc = StackTrace::GetPreviousInstructionPc(St.trace[i]);
 132       Ptr = WriteBytes(static_cast<u64>(pc), Ptr);
 133       ++Count;
 134     }
 135     // Store the count in the space we reserved earlier.
 136     *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;
 137   }
 138
 139   CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
 140         "Expected num bytes != actual bytes written");
 141 }
 142
 143 // The MIB section has the following format:
 144 // ---------- MIB Info
 145 // Num Entries
 146 // ---------- MIB Entry 0
 147 // Alloc Count
 148 // ...
 149 //       ---- AccessHistogram Entry 0
 150 //            ...
 151 //       ---- AccessHistogram Entry AccessHistogramSize - 1
 152 // ---------- MIB Entry 1
 153 // Alloc Count
 154 // ...
 155 //       ---- AccessHistogram Entry 0
 156 //            ...
 157 //       ---- AccessHistogram Entry AccessHistogramSize - 1
 158 // ----------
 159 void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,
 160                               const u64 ExpectedNumBytes, char *&Buffer) {
 161   char *Ptr = Buffer;
 162   const u64 NumEntries = StackIds.Size();
 163   Ptr = WriteBytes(NumEntries, Ptr);
 164   for (u64 i = 0; i < NumEntries; i++) {
 165     const u64 Key = StackIds[i];
 166     MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);
 167     CHECK(h.exists());
 168     Ptr = WriteBytes(Key, Ptr);
 169     // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a
 170     // serialization schema will fix this issue. See also FIXME in
 171     // deserialization.
 172     Ptr = WriteBytes((*h)->mib, Ptr);
 173     for (u64 j = 0; j < (*h)->mib.AccessHistogramSize; ++j) {
 174       u64 HistogramEntry = ((u64 *)((*h)->mib.AccessHistogram))[j];
 175       Ptr = WriteBytes(HistogramEntry, Ptr);
 176     }
 177     if ((*h)->mib.AccessHistogramSize > 0) {
 178       InternalFree((void *)((*h)->mib.AccessHistogram));
 179     }
 180   }
 181   CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&
 182         "Expected num bytes != actual bytes written");
 183 }
 184
 185 // Format
 186 // ---------- Header
 187 // Magic
 188 // Version
 189 // Total Size
 190 // Segment Offset
 191 // MIB Info Offset
 192 // Stack Offset
 193 // ---------- Segment Info
 194 // Num Entries
 195 // ---------- Segment Entry
 196 // Start
 197 // End
 198 // Offset
 199 // BuildID 32B
 200 // ----------
 201 // ...
 202 // ----------
 203 // Optional Padding Bytes
 204 // ---------- MIB Info
 205 // Num Entries
 206 // ---------- MIB Entry
 207 // Alloc Count
 208 // ...
 209 //       ---- AccessHistogram Entry 0
 210 //            ...
 211 //       ---- AccessHistogram Entry AccessHistogramSize - 1
 212 // ---------- MIB Entry 1
 213 // Alloc Count
 214 // ...
 215 //       ---- AccessHistogram Entry 0
 216 //            ...
 217 //       ---- AccessHistogram Entry AccessHistogramSize - 1
 218 // Optional Padding Bytes
 219 // ---------- Stack Info
 220 // Num Entries
 221 // ---------- Stack Entry
 222 // Num Stacks
 223 // PC1
 224 // PC2
 225 // ...
 226 // ----------
 227 // Optional Padding Bytes
 228 // ...
 229 u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,
 230                           char *&Buffer) {
 231   // Each section size is rounded up to 8b since the first entry in each section
 232   // is a u64 which holds the number of entries in the section by convention.
 233   const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Modules), 8);
 234
 235   Vector<u64> StackIds;
 236   MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds));
 237   // The first 8b are for the total number of MIB records. Each MIB record is
 238   // preceded by a 8b stack id which is associated with stack frames in the next
 239   // section.
 240   const u64 NumMIBInfoBytes = RoundUpTo(
 241       sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8);
 242
 243   // Get Number of AccessHistogram entries in total
 244   u64 TotalAccessHistogramEntries = 0;
 245   MIBMap.ForEach(
 246       [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) {
 247         u64 *TotalAccessHistogramEntries = (u64 *)Arg;
 248         *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize;
 249       },
 250       reinterpret_cast<void *>(&TotalAccessHistogramEntries));
 251   const u64 NumHistogramBytes =
 252       RoundUpTo(TotalAccessHistogramEntries * sizeof(uint64_t), 8);
 253
 254   const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8);
 255
 256   // Ensure that the profile is 8b aligned. We allow for some optional padding
 257   // at the end so that any subsequent profile serialized to the same file does
 258   // not incur unaligned accesses.
 259   const u64 TotalSizeBytes =
 260       RoundUpTo(sizeof(Header) + NumSegmentBytes + NumStackBytes +
 261                     NumMIBInfoBytes + NumHistogramBytes,
 262                 8);
 263
 264   // Allocate the memory for the entire buffer incl. info blocks.
 265   Buffer = (char *)InternalAlloc(TotalSizeBytes);
 266   char *Ptr = Buffer;
 267
 268   Header header{MEMPROF_RAW_MAGIC_64,
 269                 MEMPROF_RAW_VERSION,
 270                 static_cast<u64>(TotalSizeBytes),
 271                 sizeof(Header),
 272                 sizeof(Header) + NumSegmentBytes,
 273                 sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes +
 274                     NumHistogramBytes};
 275   Ptr = WriteBytes(header, Ptr);
 276
 277   SerializeSegmentsToBuffer(Modules, NumSegmentBytes, Ptr);
 278   Ptr += NumSegmentBytes;
 279
 280   SerializeMIBInfoToBuffer(MIBMap, StackIds,
 281                            NumMIBInfoBytes + NumHistogramBytes, Ptr);
 282   Ptr += NumMIBInfoBytes + NumHistogramBytes;
 283
 284   SerializeStackToBuffer(StackIds, NumStackBytes, Ptr);
 285
 286   return TotalSizeBytes;
 287 }
 288
 289 } // namespace __memprof