llvm/lib/Debuginfod/Debuginfod.cpp

   1 //===-- llvm/Debuginfod/Debuginfod.cpp - Debuginfod client library --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 ///
  11 /// This file contains several definitions for the debuginfod client and server.
  12 /// For the client, this file defines the fetchInfo function. For the server,
  13 /// this file defines the DebuginfodLogEntry and DebuginfodServer structs, as
  14 /// well as the DebuginfodLog, DebuginfodCollection classes. The fetchInfo
  15 /// function retrieves any of the three supported artifact types: (executable,
  16 /// debuginfo, source file) associated with a build-id from debuginfod servers.
  17 /// If a source file is to be fetched, its absolute path must be specified in
  18 /// the Description argument to fetchInfo. The DebuginfodLogEntry,
  19 /// DebuginfodLog, and DebuginfodCollection are used by the DebuginfodServer to
  20 /// scan the local filesystem for binaries and serve the debuginfod protocol.
  21 ///
  22 //===----------------------------------------------------------------------===//
  23
  24 #include "llvm/Debuginfod/Debuginfod.h"
  25 #include "llvm/ADT/StringRef.h"
  26 #include "llvm/BinaryFormat/Magic.h"
  27 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
  28 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
  29 #include "llvm/Debuginfod/HTTPClient.h"
  30 #include "llvm/Object/Binary.h"
  31 #include "llvm/Object/ELFObjectFile.h"
  32 #include "llvm/Object/ObjectFile.h"
  33 #include "llvm/Support/CachePruning.h"
  34 #include "llvm/Support/Caching.h"
  35 #include "llvm/Support/Errc.h"
  36 #include "llvm/Support/Error.h"
  37 #include "llvm/Support/FileUtilities.h"
  38 #include "llvm/Support/Path.h"
  39 #include "llvm/Support/ThreadPool.h"
  40 #include "llvm/Support/xxhash.h"
  41
  42 #include <atomic>
  43 #include <thread>
  44
  45 namespace llvm {
  46 static std::string uniqueKey(llvm::StringRef S) { return utostr(xxHash64(S)); }
  47
  48 // Returns a binary BuildID as a normalized hex string.
  49 // Uses lowercase for compatibility with common debuginfod servers.
  50 static std::string buildIDToString(BuildIDRef ID) {
  51   return llvm::toHex(ID, /*LowerCase=*/true);
  52 }
  53
  54 Expected<SmallVector<StringRef>> getDefaultDebuginfodUrls() {
  55   const char *DebuginfodUrlsEnv = std::getenv("DEBUGINFOD_URLS");
  56   if (DebuginfodUrlsEnv == nullptr)
  57     return SmallVector<StringRef>();
  58
  59   SmallVector<StringRef> DebuginfodUrls;
  60   StringRef(DebuginfodUrlsEnv).split(DebuginfodUrls, " ");
  61   return DebuginfodUrls;
  62 }
  63
  64 /// Finds a default local file caching directory for the debuginfod client,
  65 /// first checking DEBUGINFOD_CACHE_PATH.
  66 Expected<std::string> getDefaultDebuginfodCacheDirectory() {
  67   if (const char *CacheDirectoryEnv = std::getenv("DEBUGINFOD_CACHE_PATH"))
  68     return CacheDirectoryEnv;
  69
  70   SmallString<64> CacheDirectory;
  71   if (!sys::path::cache_directory(CacheDirectory))
  72     return createStringError(
  73         errc::io_error, "Unable to determine appropriate cache directory.");
  74   sys::path::append(CacheDirectory, "llvm-debuginfod", "client");
  75   return std::string(CacheDirectory);
  76 }
  77
  78 std::chrono::milliseconds getDefaultDebuginfodTimeout() {
  79   long Timeout;
  80   const char *DebuginfodTimeoutEnv = std::getenv("DEBUGINFOD_TIMEOUT");
  81   if (DebuginfodTimeoutEnv &&
  82       to_integer(StringRef(DebuginfodTimeoutEnv).trim(), Timeout, 10))
  83     return std::chrono::milliseconds(Timeout * 1000);
  84
  85   return std::chrono::milliseconds(90 * 1000);
  86 }
  87
  88 /// The following functions fetch a debuginfod artifact to a file in a local
  89 /// cache and return the cached file path. They first search the local cache,
  90 /// followed by the debuginfod servers.
  91
  92 Expected<std::string> getCachedOrDownloadSource(BuildIDRef ID,
  93                                                 StringRef SourceFilePath) {
  94   SmallString<64> UrlPath;
  95   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
  96                     buildIDToString(ID), "source",
  97                     sys::path::convert_to_slash(SourceFilePath));
  98   return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
  99 }
 100
 101 Expected<std::string> getCachedOrDownloadExecutable(BuildIDRef ID) {
 102   SmallString<64> UrlPath;
 103   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
 104                     buildIDToString(ID), "executable");
 105   return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
 106 }
 107
 108 Expected<std::string> getCachedOrDownloadDebuginfo(BuildIDRef ID) {
 109   SmallString<64> UrlPath;
 110   sys::path::append(UrlPath, sys::path::Style::posix, "buildid",
 111                     buildIDToString(ID), "debuginfo");
 112   return getCachedOrDownloadArtifact(uniqueKey(UrlPath), UrlPath);
 113 }
 114
 115 // General fetching function.
 116 Expected<std::string> getCachedOrDownloadArtifact(StringRef UniqueKey,
 117                                                   StringRef UrlPath) {
 118   SmallString<10> CacheDir;
 119
 120   Expected<std::string> CacheDirOrErr = getDefaultDebuginfodCacheDirectory();
 121   if (!CacheDirOrErr)
 122     return CacheDirOrErr.takeError();
 123   CacheDir = *CacheDirOrErr;
 124
 125   Expected<SmallVector<StringRef>> DebuginfodUrlsOrErr =
 126       getDefaultDebuginfodUrls();
 127   if (!DebuginfodUrlsOrErr)
 128     return DebuginfodUrlsOrErr.takeError();
 129   SmallVector<StringRef> &DebuginfodUrls = *DebuginfodUrlsOrErr;
 130   return getCachedOrDownloadArtifact(UniqueKey, UrlPath, CacheDir,
 131                                      DebuginfodUrls,
 132                                      getDefaultDebuginfodTimeout());
 133 }
 134
 135 namespace {
 136
 137 /// A simple handler which streams the returned data to a cache file. The cache
 138 /// file is only created if a 200 OK status is observed.
 139 class StreamedHTTPResponseHandler : public HTTPResponseHandler {
 140   using CreateStreamFn =
 141       std::function<Expected<std::unique_ptr<CachedFileStream>>()>;
 142   CreateStreamFn CreateStream;
 143   HTTPClient &Client;
 144   std::unique_ptr<CachedFileStream> FileStream;
 145
 146 public:
 147   StreamedHTTPResponseHandler(CreateStreamFn CreateStream, HTTPClient &Client)
 148       : CreateStream(CreateStream), Client(Client) {}
 149   virtual ~StreamedHTTPResponseHandler() = default;
 150
 151   Error handleBodyChunk(StringRef BodyChunk) override;
 152 };
 153
 154 } // namespace
 155
 156 Error StreamedHTTPResponseHandler::handleBodyChunk(StringRef BodyChunk) {
 157   if (!FileStream) {
 158     if (Client.responseCode() != 200)
 159       return Error::success();
 160     Expected<std::unique_ptr<CachedFileStream>> FileStreamOrError =
 161         CreateStream();
 162     if (!FileStreamOrError)
 163       return FileStreamOrError.takeError();
 164     FileStream = std::move(*FileStreamOrError);
 165   }
 166   *FileStream->OS << BodyChunk;
 167   return Error::success();
 168 }
 169
 170 Expected<std::string> getCachedOrDownloadArtifact(
 171     StringRef UniqueKey, StringRef UrlPath, StringRef CacheDirectoryPath,
 172     ArrayRef<StringRef> DebuginfodUrls, std::chrono::milliseconds Timeout) {
 173   SmallString<64> AbsCachedArtifactPath;
 174   sys::path::append(AbsCachedArtifactPath, CacheDirectoryPath,
 175                     "llvmcache-" + UniqueKey);
 176
 177   Expected<FileCache> CacheOrErr =
 178       localCache("Debuginfod-client", ".debuginfod-client", CacheDirectoryPath);
 179   if (!CacheOrErr)
 180     return CacheOrErr.takeError();
 181
 182   FileCache Cache = *CacheOrErr;
 183   // We choose an arbitrary Task parameter as we do not make use of it.
 184   unsigned Task = 0;
 185   Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, UniqueKey);
 186   if (!CacheAddStreamOrErr)
 187     return CacheAddStreamOrErr.takeError();
 188   AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
 189   if (!CacheAddStream)
 190     return std::string(AbsCachedArtifactPath);
 191   // The artifact was not found in the local cache, query the debuginfod
 192   // servers.
 193   if (!HTTPClient::isAvailable())
 194     return createStringError(errc::io_error,
 195                              "No working HTTP client is available.");
 196
 197   if (!HTTPClient::IsInitialized)
 198     return createStringError(
 199         errc::io_error,
 200         "A working HTTP client is available, but it is not initialized. To "
 201         "allow Debuginfod to make HTTP requests, call HTTPClient::initialize() "
 202         "at the beginning of main.");
 203
 204   HTTPClient Client;
 205   Client.setTimeout(Timeout);
 206   for (StringRef ServerUrl : DebuginfodUrls) {
 207     SmallString<64> ArtifactUrl;
 208     sys::path::append(ArtifactUrl, sys::path::Style::posix, ServerUrl, UrlPath);
 209
 210     // Perform the HTTP request and if successful, write the response body to
 211     // the cache.
 212     StreamedHTTPResponseHandler Handler([&]() { return CacheAddStream(Task); },
 213                                         Client);
 214     HTTPRequest Request(ArtifactUrl);
 215     Error Err = Client.perform(Request, Handler);
 216     if (Err)
 217       return std::move(Err);
 218
 219     if (Client.responseCode() != 200)
 220       continue;
 221
 222     // Return the path to the artifact on disk.
 223     return std::string(AbsCachedArtifactPath);
 224   }
 225
 226   return createStringError(errc::argument_out_of_domain, "build id not found");
 227 }
 228
 229 DebuginfodLogEntry::DebuginfodLogEntry(const Twine &Message)
 230     : Message(Message.str()) {}
 231
 232 void DebuginfodLog::push(const Twine &Message) {
 233   push(DebuginfodLogEntry(Message));
 234 }
 235
 236 void DebuginfodLog::push(DebuginfodLogEntry Entry) {
 237   {
 238     std::lock_guard<std::mutex> Guard(QueueMutex);
 239     LogEntryQueue.push(Entry);
 240   }
 241   QueueCondition.notify_one();
 242 }
 243
 244 DebuginfodLogEntry DebuginfodLog::pop() {
 245   {
 246     std::unique_lock<std::mutex> Guard(QueueMutex);
 247     // Wait for messages to be pushed into the queue.
 248     QueueCondition.wait(Guard, [&] { return !LogEntryQueue.empty(); });
 249   }
 250   std::lock_guard<std::mutex> Guard(QueueMutex);
 251   if (!LogEntryQueue.size())
 252     llvm_unreachable("Expected message in the queue.");
 253
 254   DebuginfodLogEntry Entry = LogEntryQueue.front();
 255   LogEntryQueue.pop();
 256   return Entry;
 257 }
 258
 259 DebuginfodCollection::DebuginfodCollection(ArrayRef<StringRef> PathsRef,
 260                                            DebuginfodLog &Log, ThreadPool &Pool,
 261                                            double MinInterval)
 262     : Log(Log), Pool(Pool), MinInterval(MinInterval) {
 263   for (StringRef Path : PathsRef)
 264     Paths.push_back(Path.str());
 265 }
 266
 267 Error DebuginfodCollection::update() {
 268   std::lock_guard<sys::Mutex> Guard(UpdateMutex);
 269   if (UpdateTimer.isRunning())
 270     UpdateTimer.stopTimer();
 271   UpdateTimer.clear();
 272   for (const std::string &Path : Paths) {
 273     Log.push("Updating binaries at path " + Path);
 274     if (Error Err = findBinaries(Path))
 275       return Err;
 276   }
 277   Log.push("Updated collection");
 278   UpdateTimer.startTimer();
 279   return Error::success();
 280 }
 281
 282 Expected<bool> DebuginfodCollection::updateIfStale() {
 283   if (!UpdateTimer.isRunning())
 284     return false;
 285   UpdateTimer.stopTimer();
 286   double Time = UpdateTimer.getTotalTime().getWallTime();
 287   UpdateTimer.startTimer();
 288   if (Time < MinInterval)
 289     return false;
 290   if (Error Err = update())
 291     return std::move(Err);
 292   return true;
 293 }
 294
 295 Error DebuginfodCollection::updateForever(std::chrono::milliseconds Interval) {
 296   while (true) {
 297     if (Error Err = update())
 298       return Err;
 299     std::this_thread::sleep_for(Interval);
 300   }
 301   llvm_unreachable("updateForever loop should never end");
 302 }
 303
 304 static bool isDebugBinary(object::ObjectFile *Object) {
 305   // TODO: handle PDB debuginfo
 306   std::unique_ptr<DWARFContext> Context = DWARFContext::create(
 307       *Object, DWARFContext::ProcessDebugRelocations::Process);
 308   const DWARFObject &DObj = Context->getDWARFObj();
 309   unsigned NumSections = 0;
 310   DObj.forEachInfoSections([&](const DWARFSection &S) { NumSections++; });
 311   return NumSections;
 312 }
 313
 314 static bool hasELFMagic(StringRef FilePath) {
 315   file_magic Type;
 316   std::error_code EC = identify_magic(FilePath, Type);
 317   if (EC)
 318     return false;
 319   switch (Type) {
 320   case file_magic::elf:
 321   case file_magic::elf_relocatable:
 322   case file_magic::elf_executable:
 323   case file_magic::elf_shared_object:
 324   case file_magic::elf_core:
 325     return true;
 326   default:
 327     return false;
 328   }
 329 }
 330
 331 Error DebuginfodCollection::findBinaries(StringRef Path) {
 332   std::error_code EC;
 333   sys::fs::recursive_directory_iterator I(Twine(Path), EC), E;
 334   std::mutex IteratorMutex;
 335   ThreadPoolTaskGroup IteratorGroup(Pool);
 336   for (unsigned WorkerIndex = 0; WorkerIndex < Pool.getThreadCount();
 337        WorkerIndex++) {
 338     IteratorGroup.async([&, this]() -> void {
 339       std::string FilePath;
 340       while (true) {
 341         {
 342           // Check if iteration is over or there is an error during iteration
 343           std::lock_guard<std::mutex> Guard(IteratorMutex);
 344           if (I == E || EC)
 345             return;
 346           // Grab a file path from the directory iterator and advance the
 347           // iterator.
 348           FilePath = I->path();
 349           I.increment(EC);
 350         }
 351
 352         // Inspect the file at this path to determine if it is debuginfo.
 353         if (!hasELFMagic(FilePath))
 354           continue;
 355
 356         Expected<object::OwningBinary<object::Binary>> BinOrErr =
 357             object::createBinary(FilePath);
 358
 359         if (!BinOrErr) {
 360           consumeError(BinOrErr.takeError());
 361           continue;
 362         }
 363         object::Binary *Bin = std::move(BinOrErr.get().getBinary());
 364         if (!Bin->isObject())
 365           continue;
 366
 367         // TODO: Support non-ELF binaries
 368         object::ELFObjectFileBase *Object =
 369             dyn_cast<object::ELFObjectFileBase>(Bin);
 370         if (!Object)
 371           continue;
 372
 373         Optional<BuildIDRef> ID = symbolize::getBuildID(Object);
 374         if (!ID)
 375           continue;
 376
 377         std::string IDString = buildIDToString(ID.value());
 378         if (isDebugBinary(Object)) {
 379           std::lock_guard<sys::RWMutex> DebugBinariesGuard(DebugBinariesMutex);
 380           DebugBinaries[IDString] = FilePath;
 381         } else {
 382           std::lock_guard<sys::RWMutex> BinariesGuard(BinariesMutex);
 383           Binaries[IDString] = FilePath;
 384         }
 385       }
 386     });
 387   }
 388   IteratorGroup.wait();
 389   std::unique_lock<std::mutex> Guard(IteratorMutex);
 390   if (EC)
 391     return errorCodeToError(EC);
 392   return Error::success();
 393 }
 394
 395 Expected<Optional<std::string>>
 396 DebuginfodCollection::getBinaryPath(BuildIDRef ID) {
 397   Log.push("getting binary path of ID " + buildIDToString(ID));
 398   std::shared_lock<sys::RWMutex> Guard(BinariesMutex);
 399   auto Loc = Binaries.find(buildIDToString(ID));
 400   if (Loc != Binaries.end()) {
 401     std::string Path = Loc->getValue();
 402     return Path;
 403   }
 404   return None;
 405 }
 406
 407 Expected<Optional<std::string>>
 408 DebuginfodCollection::getDebugBinaryPath(BuildIDRef ID) {
 409   Log.push("getting debug binary path of ID " + buildIDToString(ID));
 410   std::shared_lock<sys::RWMutex> Guard(DebugBinariesMutex);
 411   auto Loc = DebugBinaries.find(buildIDToString(ID));
 412   if (Loc != DebugBinaries.end()) {
 413     std::string Path = Loc->getValue();
 414     return Path;
 415   }
 416   return None;
 417 }
 418
 419 Expected<std::string> DebuginfodCollection::findBinaryPath(BuildIDRef ID) {
 420   {
 421     // Check collection; perform on-demand update if stale.
 422     Expected<Optional<std::string>> PathOrErr = getBinaryPath(ID);
 423     if (!PathOrErr)
 424       return PathOrErr.takeError();
 425     Optional<std::string> Path = *PathOrErr;
 426     if (!Path) {
 427       Expected<bool> UpdatedOrErr = updateIfStale();
 428       if (!UpdatedOrErr)
 429         return UpdatedOrErr.takeError();
 430       if (*UpdatedOrErr) {
 431         // Try once more.
 432         PathOrErr = getBinaryPath(ID);
 433         if (!PathOrErr)
 434           return PathOrErr.takeError();
 435         Path = *PathOrErr;
 436       }
 437     }
 438     if (Path)
 439       return Path.value();
 440   }
 441
 442   // Try federation.
 443   Expected<std::string> PathOrErr = getCachedOrDownloadExecutable(ID);
 444   if (!PathOrErr)
 445     consumeError(PathOrErr.takeError());
 446
 447   // Fall back to debug binary.
 448   return findDebugBinaryPath(ID);
 449 }
 450
 451 Expected<std::string> DebuginfodCollection::findDebugBinaryPath(BuildIDRef ID) {
 452   // Check collection; perform on-demand update if stale.
 453   Expected<Optional<std::string>> PathOrErr = getDebugBinaryPath(ID);
 454   if (!PathOrErr)
 455     return PathOrErr.takeError();
 456   Optional<std::string> Path = *PathOrErr;
 457   if (!Path) {
 458     Expected<bool> UpdatedOrErr = updateIfStale();
 459     if (!UpdatedOrErr)
 460       return UpdatedOrErr.takeError();
 461     if (*UpdatedOrErr) {
 462       // Try once more.
 463       PathOrErr = getBinaryPath(ID);
 464       if (!PathOrErr)
 465         return PathOrErr.takeError();
 466       Path = *PathOrErr;
 467     }
 468   }
 469   if (Path)
 470     return Path.value();
 471
 472   // Try federation.
 473   return getCachedOrDownloadDebuginfo(ID);
 474 }
 475
 476 DebuginfodServer::DebuginfodServer(DebuginfodLog &Log,
 477                                    DebuginfodCollection &Collection)
 478     : Log(Log), Collection(Collection) {
 479   cantFail(
 480       Server.get(R"(/buildid/(.*)/debuginfo)", [&](HTTPServerRequest Request) {
 481         Log.push("GET " + Request.UrlPath);
 482         std::string IDString;
 483         if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) {
 484           Request.setResponse(
 485               {404, "text/plain", "Build ID is not a hex string\n"});
 486           return;
 487         }
 488         BuildID ID(IDString.begin(), IDString.end());
 489         Expected<std::string> PathOrErr = Collection.findDebugBinaryPath(ID);
 490         if (Error Err = PathOrErr.takeError()) {
 491           consumeError(std::move(Err));
 492           Request.setResponse({404, "text/plain", "Build ID not found\n"});
 493           return;
 494         }
 495         streamFile(Request, *PathOrErr);
 496       }));
 497   cantFail(
 498       Server.get(R"(/buildid/(.*)/executable)", [&](HTTPServerRequest Request) {
 499         Log.push("GET " + Request.UrlPath);
 500         std::string IDString;
 501         if (!tryGetFromHex(Request.UrlPathMatches[0], IDString)) {
 502           Request.setResponse(
 503               {404, "text/plain", "Build ID is not a hex string\n"});
 504           return;
 505         }
 506         BuildID ID(IDString.begin(), IDString.end());
 507         Expected<std::string> PathOrErr = Collection.findBinaryPath(ID);
 508         if (Error Err = PathOrErr.takeError()) {
 509           consumeError(std::move(Err));
 510           Request.setResponse({404, "text/plain", "Build ID not found\n"});
 511           return;
 512         }
 513         streamFile(Request, *PathOrErr);
 514       }));
 515 }
 516
 517 } // namespace llvm