tools/android/memdump/memdump.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <fcntl.h>
   6 #include <signal.h>
   7 #include <sys/types.h>
   8 #include <unistd.h>
   9
  10 #include <algorithm>
  11 #include <cstring>
  12 #include <fstream>
  13 #include <iostream>
  14 #include <limits>
  15 #include <string>
  16 #include <utility>
  17 #include <vector>
  18
  19 #include "base/base64.h"
  20 #include "base/basictypes.h"
  21 #include "base/bind.h"
  22 #include "base/callback_helpers.h"
  23 #include "base/containers/hash_tables.h"
  24 #include "base/files/file_util.h"
  25 #include "base/files/scoped_file.h"
  26 #include "base/format_macros.h"
  27 #include "base/logging.h"
  28 #include "base/strings/string_number_conversions.h"
  29 #include "base/strings/string_split.h"
  30 #include "base/strings/stringprintf.h"
  31
  32 const unsigned int kPageSize = getpagesize();
  33
  34 namespace {
  35
  36 class BitSet {
  37  public:
  38   void resize(size_t nbits) {
  39     data_.resize((nbits + 7) / 8);
  40   }
  41
  42   void set(uint32 bit) {
  43     const uint32 byte_idx = bit / 8;
  44     CHECK(byte_idx < data_.size());
  45     data_[byte_idx] |= (1 << (bit & 7));
  46   }
  47
  48   std::string AsB64String() const {
  49     // Simple optimization: strip trailing zero bytes from the bitmap.
  50     // For instance, if a region has 32 pages but only the first 9 are resident,
  51     // The full bitmap would be 0xff 0x01 0x00 0x00, the stripped one 0xff 0x01.
  52     // It can save up to some seconds when printing large mmaps, in particular
  53     // in presence of large virtual address space reservations (where none of
  54     // the pages are resident).
  55     size_t end = data_.size();
  56     while (end > 0 && data_[end - 1] == '\0')
  57       --end;
  58     std::string bits(&data_[0], end);
  59     std::string b64_string;
  60     base::Base64Encode(bits, &b64_string);
  61     return b64_string;
  62   }
  63
  64  private:
  65   std::vector<char> data_;
  66 };
  67
  68 // An entry in /proc/<pid>/pagemap.
  69 struct PageMapEntry {
  70   uint64 page_frame_number : 55;
  71   uint unused : 8;
  72   uint present : 1;
  73 };
  74
  75 // Describes a memory page.
  76 struct PageInfo {
  77   int64 page_frame_number; // Physical page id, also known as PFN.
  78   int64 flags;
  79   int32 times_mapped;
  80 };
  81
  82 struct PageCount {
  83   PageCount() : total_count(0), unevictable_count(0) {}
  84
  85   int total_count;
  86   int unevictable_count;
  87 };
  88
  89 struct MemoryMap {
  90   std::string name;
  91   std::string flags;
  92   uint64 start_address;
  93   uint64 end_address;
  94   uint64 offset;
  95   PageCount private_pages;
  96   // app_shared_pages[i] contains the number of pages mapped in i+2 processes
  97   // (only among the processes that are being analyzed).
  98   std::vector<PageCount> app_shared_pages;
  99   PageCount other_shared_pages;
 100   std::vector<PageInfo> committed_pages;
 101   // committed_pages_bits is a bitset reflecting the present bit for all the
 102   // virtual pages of the mapping.
 103   BitSet committed_pages_bits;
 104 };
 105
 106 struct ProcessMemory {
 107   pid_t pid;
 108   std::vector<MemoryMap> memory_maps;
 109 };
 110
 111 bool PageIsUnevictable(const PageInfo& page_info) {
 112   // These constants are taken from kernel-page-flags.h.
 113   const int KPF_DIRTY = 4; // Note that only file-mapped pages can be DIRTY.
 114   const int KPF_ANON = 12; // Anonymous pages are dirty per definition.
 115   const int KPF_UNEVICTABLE = 18;
 116   const int KPF_MLOCKED = 33;
 117
 118   return (page_info.flags & ((1ll << KPF_DIRTY) |
 119                              (1ll << KPF_ANON) |
 120                              (1ll << KPF_UNEVICTABLE) |
 121                              (1ll << KPF_MLOCKED))) ?
 122                              true : false;
 123 }
 124
 125 // Number of times a physical page is mapped in a process.
 126 typedef base::hash_map<uint64, int> PFNMap;
 127
 128 // Parses lines from /proc/<PID>/maps, e.g.:
 129 // 401e7000-401f5000 r-xp 00000000 103:02 158       /system/bin/linker
 130 bool ParseMemoryMapLine(const std::string& line,
 131                         std::vector<std::string>* tokens,
 132                         MemoryMap* memory_map) {
 133   tokens->clear();
 134   base::SplitString(line, ' ', tokens);
 135   if (tokens->size() < 2)
 136     return false;
 137   const std::string& addr_range = tokens->at(0);
 138   std::vector<std::string> range_tokens;
 139   base::SplitString(addr_range, '-', &range_tokens);
 140   const std::string& start_address_token = range_tokens.at(0);
 141   if (!base::HexStringToUInt64(start_address_token,
 142                                &memory_map->start_address)) {
 143     return false;
 144   }
 145   const std::string& end_address_token = range_tokens.at(1);
 146   if (!base::HexStringToUInt64(end_address_token, &memory_map->end_address)) {
 147     return false;
 148   }
 149   if (tokens->at(1).size() != strlen("rwxp"))
 150     return false;
 151   memory_map->flags.swap(tokens->at(1));
 152   if (!base::HexStringToUInt64(tokens->at(2), &memory_map->offset))
 153     return false;
 154   memory_map->committed_pages_bits.resize(
 155       (memory_map->end_address - memory_map->start_address) / kPageSize);
 156   const int map_name_index = 5;
 157   if (tokens->size() >= map_name_index + 1) {
 158     for (std::vector<std::string>::const_iterator it =
 159              tokens->begin() + map_name_index; it != tokens->end(); ++it) {
 160       if (!it->empty()) {
 161         if (!memory_map->name.empty())
 162           memory_map->name.append(" ");
 163         memory_map->name.append(*it);
 164       }
 165     }
 166   }
 167   return true;
 168 }
 169
 170 // Reads sizeof(T) bytes from file |fd| at |offset|.
 171 template <typename T>
 172 bool ReadFromFileAtOffset(int fd, off_t offset, T* value) {
 173   if (lseek64(fd, offset * sizeof(*value), SEEK_SET) < 0) {
 174     PLOG(ERROR) << "lseek";
 175     return false;
 176   }
 177   ssize_t bytes = read(fd, value, sizeof(*value));
 178   if (bytes != sizeof(*value) && bytes != 0) {
 179     PLOG(ERROR) << "read";
 180     return false;
 181   }
 182   return true;
 183 }
 184
 185 // Fills |process_maps| in with the process memory maps identified by |pid|.
 186 bool GetProcessMaps(pid_t pid, std::vector<MemoryMap>* process_maps) {
 187   std::ifstream maps_file(base::StringPrintf("/proc/%d/maps", pid).c_str());
 188   if (!maps_file.good()) {
 189     PLOG(ERROR) << "open";
 190     return false;
 191   }
 192   std::string line;
 193   std::vector<std::string> tokens;
 194   while (std::getline(maps_file, line) && !line.empty()) {
 195     MemoryMap memory_map = {};
 196     if (!ParseMemoryMapLine(line, &tokens, &memory_map)) {
 197       LOG(ERROR) << "Could not parse line: " << line;
 198       return false;
 199     }
 200     process_maps->push_back(memory_map);
 201   }
 202   return true;
 203 }
 204
 205 // Fills |committed_pages| in with the set of committed pages contained in the
 206 // provided memory map.
 207 bool GetPagesForMemoryMap(int pagemap_fd,
 208                           const MemoryMap& memory_map,
 209                           std::vector<PageInfo>* committed_pages,
 210                           BitSet* committed_pages_bits) {
 211   const off64_t offset = memory_map.start_address / kPageSize;
 212   if (lseek64(pagemap_fd, offset * sizeof(PageMapEntry), SEEK_SET) < 0) {
 213     PLOG(ERROR) << "lseek";
 214     return false;
 215   }
 216   for (uint64 addr = memory_map.start_address, page_index = 0;
 217        addr < memory_map.end_address;
 218        addr += kPageSize, ++page_index) {
 219     DCHECK_EQ(0, addr % kPageSize);
 220     PageMapEntry page_map_entry = {};
 221     static_assert(sizeof(PageMapEntry) == sizeof(uint64), "unexpected size");
 222     ssize_t bytes = read(pagemap_fd, &page_map_entry, sizeof(page_map_entry));
 223     if (bytes != sizeof(PageMapEntry) && bytes != 0) {
 224       PLOG(ERROR) << "read";
 225       return false;
 226     }
 227     if (page_map_entry.present) {  // Ignore non-committed pages.
 228       if (page_map_entry.page_frame_number == 0)
 229         continue;
 230       PageInfo page_info = {};
 231       page_info.page_frame_number = page_map_entry.page_frame_number;
 232       committed_pages->push_back(page_info);
 233       committed_pages_bits->set(page_index);
 234     }
 235   }
 236   return true;
 237 }
 238
 239 // Fills |committed_pages| with mapping count and flags information gathered
 240 // looking-up /proc/kpagecount and /proc/kpageflags.
 241 bool SetPagesInfo(int pagecount_fd,
 242                   int pageflags_fd,
 243                   std::vector<PageInfo>* pages) {
 244   for (std::vector<PageInfo>::iterator it = pages->begin();
 245        it != pages->end(); ++it) {
 246     PageInfo* const page_info = &*it;
 247     int64 times_mapped;
 248     if (!ReadFromFileAtOffset(
 249             pagecount_fd, page_info->page_frame_number, &times_mapped)) {
 250       return false;
 251     }
 252     DCHECK(times_mapped <= std::numeric_limits<int32_t>::max());
 253     page_info->times_mapped = static_cast<int32>(times_mapped);
 254
 255     int64 page_flags;
 256     if (!ReadFromFileAtOffset(
 257             pageflags_fd, page_info->page_frame_number, &page_flags)) {
 258       return false;
 259     }
 260     page_info->flags = page_flags;
 261   }
 262   return true;
 263 }
 264
 265 // Fills in the provided vector of Page Frame Number maps. This lets
 266 // ClassifyPages() know how many times each page is mapped in the processes.
 267 void FillPFNMaps(const std::vector<ProcessMemory>& processes_memory,
 268                  std::vector<PFNMap>* pfn_maps) {
 269   int current_process_index = 0;
 270   for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
 271        it != processes_memory.end(); ++it, ++current_process_index) {
 272     const std::vector<MemoryMap>& memory_maps = it->memory_maps;
 273     for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
 274          it != memory_maps.end(); ++it) {
 275       const std::vector<PageInfo>& pages = it->committed_pages;
 276       for (std::vector<PageInfo>::const_iterator it = pages.begin();
 277            it != pages.end(); ++it) {
 278         const PageInfo& page_info = *it;
 279         PFNMap* const pfn_map = &(*pfn_maps)[current_process_index];
 280         const std::pair<PFNMap::iterator, bool> result = pfn_map->insert(
 281             std::make_pair(page_info.page_frame_number, 0));
 282         ++result.first->second;
 283       }
 284     }
 285   }
 286 }
 287
 288 // Sets the private_count/app_shared_pages/other_shared_count fields of the
 289 // provided memory maps for each process.
 290 void ClassifyPages(std::vector<ProcessMemory>* processes_memory) {
 291   std::vector<PFNMap> pfn_maps(processes_memory->size());
 292   FillPFNMaps(*processes_memory, &pfn_maps);
 293   // Hash set keeping track of the physical pages mapped in a single process so
 294   // that they can be counted only once.
 295   base::hash_set<uint64> physical_pages_mapped_in_process;
 296
 297   for (std::vector<ProcessMemory>::iterator it = processes_memory->begin();
 298        it != processes_memory->end(); ++it) {
 299     std::vector<MemoryMap>* const memory_maps = &it->memory_maps;
 300     physical_pages_mapped_in_process.clear();
 301     for (std::vector<MemoryMap>::iterator it = memory_maps->begin();
 302          it != memory_maps->end(); ++it) {
 303       MemoryMap* const memory_map = &*it;
 304       const size_t processes_count = processes_memory->size();
 305       memory_map->app_shared_pages.resize(processes_count - 1);
 306       const std::vector<PageInfo>& pages = memory_map->committed_pages;
 307       for (std::vector<PageInfo>::const_iterator it = pages.begin();
 308            it != pages.end(); ++it) {
 309         const PageInfo& page_info = *it;
 310         if (page_info.times_mapped == 1) {
 311           ++memory_map->private_pages.total_count;
 312           if (PageIsUnevictable(page_info))
 313             ++memory_map->private_pages.unevictable_count;
 314           continue;
 315         }
 316         const uint64 page_frame_number = page_info.page_frame_number;
 317         const std::pair<base::hash_set<uint64>::iterator, bool> result =
 318             physical_pages_mapped_in_process.insert(page_frame_number);
 319         const bool did_insert = result.second;
 320         if (!did_insert) {
 321           // This physical page (mapped multiple times in the same process) was
 322           // already counted.
 323           continue;
 324         }
 325         // See if the current physical page is also mapped in the other
 326         // processes that are being analyzed.
 327         int times_mapped = 0;
 328         int mapped_in_processes_count = 0;
 329         for (std::vector<PFNMap>::const_iterator it = pfn_maps.begin();
 330              it != pfn_maps.end(); ++it) {
 331           const PFNMap& pfn_map = *it;
 332           const PFNMap::const_iterator found_it = pfn_map.find(
 333               page_frame_number);
 334           if (found_it == pfn_map.end())
 335             continue;
 336           ++mapped_in_processes_count;
 337           times_mapped += found_it->second;
 338         }
 339         PageCount* page_count_to_update = NULL;
 340         if (times_mapped == page_info.times_mapped) {
 341           // The physical page is only mapped in the processes that are being
 342           // analyzed.
 343           if (mapped_in_processes_count > 1) {
 344             // The physical page is mapped in multiple processes.
 345             page_count_to_update =
 346                 &memory_map->app_shared_pages[mapped_in_processes_count - 2];
 347           } else {
 348             // The physical page is mapped multiple times in the same process.
 349             page_count_to_update = &memory_map->private_pages;
 350           }
 351         } else {
 352           page_count_to_update = &memory_map->other_shared_pages;
 353         }
 354         ++page_count_to_update->total_count;
 355         if (PageIsUnevictable(page_info))
 356           ++page_count_to_update->unevictable_count;
 357       }
 358     }
 359   }
 360 }
 361
 362 void AppendAppSharedField(const std::vector<PageCount>& app_shared_pages,
 363                           std::string* out) {
 364   out->append("[");
 365   for (std::vector<PageCount>::const_iterator it = app_shared_pages.begin();
 366        it != app_shared_pages.end(); ++it) {
 367     out->append(base::IntToString(it->total_count * kPageSize));
 368     out->append(":");
 369     out->append(base::IntToString(it->unevictable_count * kPageSize));
 370     if (it + 1 != app_shared_pages.end())
 371       out->append(",");
 372   }
 373   out->append("]");
 374 }
 375
 376 void DumpProcessesMemoryMapsInShortFormat(
 377     const std::vector<ProcessMemory>& processes_memory) {
 378   const int KB_PER_PAGE = kPageSize >> 10;
 379   std::vector<int> totals_app_shared(processes_memory.size());
 380   std::string buf;
 381   std::cout << "pid\tprivate\t\tshared_app\tshared_other (KB)\n";
 382   for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
 383        it != processes_memory.end(); ++it) {
 384     const ProcessMemory& process_memory = *it;
 385     std::fill(totals_app_shared.begin(), totals_app_shared.end(), 0);
 386     int total_private = 0, total_other_shared = 0;
 387     const std::vector<MemoryMap>& memory_maps = process_memory.memory_maps;
 388     for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
 389          it != memory_maps.end(); ++it) {
 390       const MemoryMap& memory_map = *it;
 391       total_private += memory_map.private_pages.total_count;
 392       for (size_t i = 0; i < memory_map.app_shared_pages.size(); ++i)
 393         totals_app_shared[i] += memory_map.app_shared_pages[i].total_count;
 394       total_other_shared += memory_map.other_shared_pages.total_count;
 395     }
 396     double total_app_shared = 0;
 397     for (size_t i = 0; i < totals_app_shared.size(); ++i)
 398       total_app_shared += static_cast<double>(totals_app_shared[i]) / (i + 2);
 399     base::SStringPrintf(
 400         &buf, "%d\t%d\t\t%d\t\t%d\n",
 401         process_memory.pid,
 402         total_private * KB_PER_PAGE,
 403         static_cast<int>(total_app_shared) * KB_PER_PAGE,
 404         total_other_shared * KB_PER_PAGE);
 405     std::cout << buf;
 406   }
 407 }
 408
 409 void DumpProcessesMemoryMapsInExtendedFormat(
 410     const std::vector<ProcessMemory>& processes_memory) {
 411   std::string buf;
 412   std::string app_shared_buf;
 413   for (std::vector<ProcessMemory>::const_iterator it = processes_memory.begin();
 414        it != processes_memory.end(); ++it) {
 415     const ProcessMemory& process_memory = *it;
 416     std::cout << "[ PID=" << process_memory.pid << "]" << '\n';
 417     const std::vector<MemoryMap>& memory_maps = process_memory.memory_maps;
 418     for (std::vector<MemoryMap>::const_iterator it = memory_maps.begin();
 419          it != memory_maps.end(); ++it) {
 420       const MemoryMap& memory_map = *it;
 421       app_shared_buf.clear();
 422       AppendAppSharedField(memory_map.app_shared_pages, &app_shared_buf);
 423       base::SStringPrintf(
 424           &buf,
 425           "%"PRIx64"-%"PRIx64" %s %"PRIx64" private_unevictable=%d private=%d "
 426           "shared_app=%s shared_other_unevictable=%d shared_other=%d "
 427           "\"%s\" [%s]\n",
 428           memory_map.start_address,
 429           memory_map.end_address,
 430           memory_map.flags.c_str(),
 431           memory_map.offset,
 432           memory_map.private_pages.unevictable_count * kPageSize,
 433           memory_map.private_pages.total_count * kPageSize,
 434           app_shared_buf.c_str(),
 435           memory_map.other_shared_pages.unevictable_count * kPageSize,
 436           memory_map.other_shared_pages.total_count * kPageSize,
 437           memory_map.name.c_str(),
 438           memory_map.committed_pages_bits.AsB64String().c_str());
 439       std::cout << buf;
 440     }
 441   }
 442 }
 443
 444 bool CollectProcessMemoryInformation(int page_count_fd,
 445                                      int page_flags_fd,
 446                                      ProcessMemory* process_memory) {
 447   const pid_t pid = process_memory->pid;
 448   base::ScopedFD pagemap_fd(HANDLE_EINTR(open(
 449       base::StringPrintf("/proc/%d/pagemap", pid).c_str(), O_RDONLY)));
 450   if (!pagemap_fd.is_valid()) {
 451     PLOG(ERROR) << "open";
 452     return false;
 453   }
 454   std::vector<MemoryMap>* const process_maps = &process_memory->memory_maps;
 455   if (!GetProcessMaps(pid, process_maps))
 456     return false;
 457   for (std::vector<MemoryMap>::iterator it = process_maps->begin();
 458        it != process_maps->end(); ++it) {
 459     std::vector<PageInfo>* const committed_pages = &it->committed_pages;
 460     BitSet* const pages_bits = &it->committed_pages_bits;
 461     GetPagesForMemoryMap(pagemap_fd.get(), *it, committed_pages, pages_bits);
 462     SetPagesInfo(page_count_fd, page_flags_fd, committed_pages);
 463   }
 464   return true;
 465 }
 466
 467 void KillAll(const std::vector<pid_t>& pids, int signal_number) {
 468   for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
 469        ++it) {
 470     kill(*it, signal_number);
 471   }
 472 }
 473
 474 void ExitWithUsage() {
 475   LOG(ERROR) << "Usage: memdump [-a] <PID1>... <PIDN>";
 476   exit(EXIT_FAILURE);
 477 }
 478
 479 }  // namespace
 480
 481 int main(int argc, char** argv) {
 482   if (argc == 1)
 483     ExitWithUsage();
 484   const bool short_output = !strncmp(argv[1], "-a", 2);
 485   if (short_output) {
 486     if (argc == 2)
 487       ExitWithUsage();
 488     ++argv;
 489   }
 490   std::vector<pid_t> pids;
 491   for (const char* const* ptr = argv + 1; *ptr; ++ptr) {
 492     pid_t pid;
 493     if (!base::StringToInt(*ptr, &pid))
 494       return EXIT_FAILURE;
 495     pids.push_back(pid);
 496   }
 497
 498   std::vector<ProcessMemory> processes_memory(pids.size());
 499   {
 500     base::ScopedFD page_count_fd(
 501         HANDLE_EINTR(open("/proc/kpagecount", O_RDONLY)));
 502     if (!page_count_fd.is_valid()) {
 503       PLOG(ERROR) << "open /proc/kpagecount";
 504       return EXIT_FAILURE;
 505     }
 506
 507     base::ScopedFD page_flags_fd(open("/proc/kpageflags", O_RDONLY));
 508     if (!page_flags_fd.is_valid()) {
 509       PLOG(ERROR) << "open /proc/kpageflags";
 510       return EXIT_FAILURE;
 511     }
 512
 513     base::ScopedClosureRunner auto_resume_processes(
 514         base::Bind(&KillAll, pids, SIGCONT));
 515     KillAll(pids, SIGSTOP);
 516     for (std::vector<pid_t>::const_iterator it = pids.begin(); it != pids.end();
 517          ++it) {
 518       ProcessMemory* const process_memory =
 519           &processes_memory[it - pids.begin()];
 520       process_memory->pid = *it;
 521       if (!CollectProcessMemoryInformation(
 522               page_count_fd.get(), page_flags_fd.get(), process_memory)) {
 523         return EXIT_FAILURE;
 524       }
 525     }
 526   }
 527
 528   ClassifyPages(&processes_memory);
 529   if (short_output)
 530     DumpProcessesMemoryMapsInShortFormat(processes_memory);
 531   else
 532     DumpProcessesMemoryMapsInExtendedFormat(processes_memory);
 533   return EXIT_SUCCESS;
 534 }