lldb/source/Utility/Reproducer.cpp

   1 //===-- Reproducer.cpp ----------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "lldb/Utility/Reproducer.h"
  10 #include "lldb/Utility/LLDBAssert.h"
  11 #include "lldb/Utility/ReproducerProvider.h"
  12 #include "lldb/Utility/Timer.h"
  13
  14 #include "llvm/Support/FileSystem.h"
  15 #include "llvm/Support/Threading.h"
  16 #include "llvm/Support/raw_ostream.h"
  17
  18 using namespace lldb_private;
  19 using namespace lldb_private::repro;
  20 using namespace llvm;
  21 using namespace llvm::yaml;
  22
  23 Reproducer &Reproducer::Instance() { return *InstanceImpl(); }
  24
  25 llvm::Error Reproducer::Initialize(ReproducerMode mode,
  26                                    llvm::Optional<FileSpec> root) {
  27   lldbassert(!InstanceImpl() && "Already initialized.");
  28   InstanceImpl().emplace();
  29
  30   switch (mode) {
  31   case ReproducerMode::Capture: {
  32     if (!root) {
  33       SmallString<128> repro_dir;
  34       auto ec = sys::fs::createUniqueDirectory("reproducer", repro_dir);
  35       if (ec)
  36         return make_error<StringError>(
  37             "unable to create unique reproducer directory", ec);
  38       root.emplace(repro_dir);
  39     } else {
  40       auto ec = sys::fs::create_directory(root->GetPath());
  41       if (ec)
  42         return make_error<StringError>("unable to create reproducer directory",
  43                                        ec);
  44     }
  45     return Instance().SetCapture(root);
  46   } break;
  47   case ReproducerMode::Off:
  48     break;
  49   };
  50
  51   return Error::success();
  52 }
  53
  54 void Reproducer::Initialize() {
  55   llvm::cantFail(Initialize(repro::ReproducerMode::Off, llvm::None));
  56 }
  57
  58 bool Reproducer::Initialized() { return InstanceImpl().operator bool(); }
  59
  60 void Reproducer::Terminate() {
  61   lldbassert(InstanceImpl() && "Already terminated.");
  62   InstanceImpl().reset();
  63 }
  64
  65 Optional<Reproducer> &Reproducer::InstanceImpl() {
  66   static Optional<Reproducer> g_reproducer;
  67   return g_reproducer;
  68 }
  69
  70 const Generator *Reproducer::GetGenerator() const {
  71   std::lock_guard<std::mutex> guard(m_mutex);
  72   if (m_generator)
  73     return &(*m_generator);
  74   return nullptr;
  75 }
  76
  77 const Loader *Reproducer::GetLoader() const {
  78   std::lock_guard<std::mutex> guard(m_mutex);
  79   if (m_loader)
  80     return &(*m_loader);
  81   return nullptr;
  82 }
  83
  84 Generator *Reproducer::GetGenerator() {
  85   std::lock_guard<std::mutex> guard(m_mutex);
  86   if (m_generator)
  87     return &(*m_generator);
  88   return nullptr;
  89 }
  90
  91 Loader *Reproducer::GetLoader() {
  92   std::lock_guard<std::mutex> guard(m_mutex);
  93   if (m_loader)
  94     return &(*m_loader);
  95   return nullptr;
  96 }
  97
  98 llvm::Error Reproducer::SetCapture(llvm::Optional<FileSpec> root) {
  99   std::lock_guard<std::mutex> guard(m_mutex);
 100
 101   if (root && m_loader)
 102     return make_error<StringError>(
 103         "cannot generate a reproducer when replay one",
 104         inconvertibleErrorCode());
 105
 106   if (!root) {
 107     m_generator.reset();
 108     return Error::success();
 109   }
 110
 111   m_generator.emplace(*root);
 112   return Error::success();
 113 }
 114
 115 FileSpec Reproducer::GetReproducerPath() const {
 116   if (auto g = GetGenerator())
 117     return g->GetRoot();
 118   if (auto l = GetLoader())
 119     return l->GetRoot();
 120   return {};
 121 }
 122
 123 static FileSpec MakeAbsolute(const FileSpec &file_spec) {
 124   SmallString<128> path;
 125   file_spec.GetPath(path, false);
 126   llvm::sys::fs::make_absolute(path);
 127   return FileSpec(path, file_spec.GetPathStyle());
 128 }
 129
 130 Generator::Generator(FileSpec root) : m_root(MakeAbsolute(std::move(root))) {
 131   GetOrCreate<repro::WorkingDirectoryProvider>();
 132   GetOrCreate<repro::HomeDirectoryProvider>();
 133 }
 134
 135 Generator::~Generator() {
 136   if (!m_done) {
 137     if (m_auto_generate) {
 138       Keep();
 139       llvm::cantFail(Finalize(GetRoot()));
 140     } else {
 141       Discard();
 142     }
 143   }
 144 }
 145
 146 ProviderBase *Generator::Register(std::unique_ptr<ProviderBase> provider) {
 147   std::lock_guard<std::mutex> lock(m_providers_mutex);
 148   std::pair<const void *, std::unique_ptr<ProviderBase>> key_value(
 149       provider->DynamicClassID(), std::move(provider));
 150   auto e = m_providers.insert(std::move(key_value));
 151   return e.first->getSecond().get();
 152 }
 153
 154 void Generator::Keep() {
 155   LLDB_SCOPED_TIMER();
 156   assert(!m_done);
 157   m_done = true;
 158
 159   for (auto &provider : m_providers)
 160     provider.second->Keep();
 161
 162   AddProvidersToIndex();
 163 }
 164
 165 void Generator::Discard() {
 166   LLDB_SCOPED_TIMER();
 167   assert(!m_done);
 168   m_done = true;
 169
 170   for (auto &provider : m_providers)
 171     provider.second->Discard();
 172
 173   llvm::sys::fs::remove_directories(m_root.GetPath());
 174 }
 175
 176 void Generator::SetAutoGenerate(bool b) { m_auto_generate = b; }
 177
 178 bool Generator::IsAutoGenerate() const { return m_auto_generate; }
 179
 180 const FileSpec &Generator::GetRoot() const { return m_root; }
 181
 182 void Generator::AddProvidersToIndex() {
 183   FileSpec index = m_root;
 184   index.AppendPathComponent("index.yaml");
 185
 186   std::error_code EC;
 187   auto strm = std::make_unique<raw_fd_ostream>(index.GetPath(), EC,
 188                                                sys::fs::OpenFlags::OF_None);
 189   yaml::Output yout(*strm);
 190
 191   std::vector<std::string> files;
 192   files.reserve(m_providers.size());
 193   for (auto &provider : m_providers) {
 194     files.emplace_back(provider.second->GetFile());
 195   }
 196
 197   yout << files;
 198 }
 199
 200 Loader::Loader(FileSpec root, bool passive)
 201     : m_root(MakeAbsolute(std::move(root))), m_loaded(false) {}
 202
 203 llvm::Error Loader::LoadIndex() {
 204   if (m_loaded)
 205     return llvm::Error::success();
 206
 207   FileSpec index = m_root.CopyByAppendingPathComponent("index.yaml");
 208
 209   auto error_or_file = MemoryBuffer::getFile(index.GetPath());
 210   if (auto err = error_or_file.getError())
 211     return make_error<StringError>("unable to load reproducer index", err);
 212
 213   yaml::Input yin((*error_or_file)->getBuffer());
 214   yin >> m_files;
 215   if (auto err = yin.error())
 216     return make_error<StringError>("unable to read reproducer index", err);
 217
 218   // Sort files to speed up search.
 219   llvm::sort(m_files);
 220
 221   // Remember that we've loaded the index.
 222   m_loaded = true;
 223
 224   return llvm::Error::success();
 225 }
 226
 227 bool Loader::HasFile(StringRef file) {
 228   assert(m_loaded);
 229   auto it = std::lower_bound(m_files.begin(), m_files.end(), file.str());
 230   return (it != m_files.end()) && (*it == file);
 231 }
 232
 233 void Verifier::Verify(
 234     llvm::function_ref<void(llvm::StringRef)> error_callback,
 235     llvm::function_ref<void(llvm::StringRef)> warning_callback,
 236     llvm::function_ref<void(llvm::StringRef)> note_callack) const {
 237   if (!m_loader) {
 238     error_callback("invalid loader");
 239     return;
 240   }
 241
 242   FileSpec vfs_mapping = m_loader->GetFile<FileProvider::Info>();
 243   ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
 244       vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath());
 245   if (!buffer) {
 246     error_callback("unable to read files: " + buffer.getError().message());
 247     return;
 248   }
 249
 250   IntrusiveRefCntPtr<vfs::FileSystem> vfs = vfs::getVFSFromYAML(
 251       std::move(buffer.get()), nullptr, vfs_mapping.GetPath());
 252   if (!vfs) {
 253     error_callback("unable to initialize the virtual file system");
 254     return;
 255   }
 256
 257   auto &redirecting_vfs = static_cast<vfs::RedirectingFileSystem &>(*vfs);
 258   redirecting_vfs.setFallthrough(false);
 259
 260   {
 261     llvm::Expected<std::string> working_dir =
 262         GetDirectoryFrom<WorkingDirectoryProvider>(m_loader);
 263     if (working_dir) {
 264       if (!vfs->exists(*working_dir))
 265         warning_callback("working directory '" + *working_dir + "' not in VFS");
 266       vfs->setCurrentWorkingDirectory(*working_dir);
 267     } else {
 268       warning_callback("no working directory in reproducer: " +
 269                        toString(working_dir.takeError()));
 270     }
 271   }
 272
 273   {
 274     llvm::Expected<std::string> home_dir =
 275         GetDirectoryFrom<HomeDirectoryProvider>(m_loader);
 276     if (home_dir) {
 277       if (!vfs->exists(*home_dir))
 278         warning_callback("home directory '" + *home_dir + "' not in VFS");
 279     } else {
 280       warning_callback("no home directory in reproducer: " +
 281                        toString(home_dir.takeError()));
 282     }
 283   }
 284
 285   {
 286     Expected<std::string> symbol_files =
 287         m_loader->LoadBuffer<SymbolFileProvider>();
 288     if (symbol_files) {
 289       std::vector<SymbolFileProvider::Entry> entries;
 290       llvm::yaml::Input yin(*symbol_files);
 291       yin >> entries;
 292       for (const auto &entry : entries) {
 293         if (!entry.module_path.empty() && !vfs->exists(entry.module_path)) {
 294           warning_callback("'" + entry.module_path + "': module path for " +
 295                            entry.uuid + " not in VFS");
 296         }
 297         if (!entry.symbol_path.empty() && !vfs->exists(entry.symbol_path)) {
 298           warning_callback("'" + entry.symbol_path + "': symbol path for " +
 299                            entry.uuid + " not in VFS");
 300         }
 301       }
 302     } else {
 303       llvm::consumeError(symbol_files.takeError());
 304     }
 305   }
 306
 307   // Missing files in the VFS are notes rather than warnings. Because the VFS
 308   // is a snapshot, temporary files could have been removed between when they
 309   // were recorded and when the reproducer was generated.
 310   std::vector<llvm::StringRef> roots = redirecting_vfs.getRoots();
 311   for (llvm::StringRef root : roots) {
 312     std::error_code ec;
 313     vfs::recursive_directory_iterator iter(*vfs, root, ec);
 314     vfs::recursive_directory_iterator end;
 315     for (; iter != end && !ec; iter.increment(ec)) {
 316       ErrorOr<vfs::Status> status = vfs->status(iter->path());
 317       if (!status)
 318         note_callack("'" + iter->path().str() +
 319                      "': " + status.getError().message());
 320     }
 321   }
 322 }
 323
 324 static llvm::Error addPaths(StringRef path,
 325                             function_ref<void(StringRef)> callback) {
 326   auto buffer = llvm::MemoryBuffer::getFile(path);
 327   if (!buffer)
 328     return errorCodeToError(buffer.getError());
 329
 330   SmallVector<StringRef, 0> paths;
 331   (*buffer)->getBuffer().split(paths, '\0');
 332   for (StringRef p : paths) {
 333     if (!p.empty() && llvm::sys::fs::exists(p))
 334       callback(p);
 335   }
 336
 337   return errorCodeToError(llvm::sys::fs::remove(path));
 338 }
 339
 340 llvm::Error repro::Finalize(Loader *loader) {
 341   if (!loader)
 342     return make_error<StringError>("invalid loader",
 343                                    llvm::inconvertibleErrorCode());
 344
 345   FileSpec reproducer_root = loader->GetRoot();
 346   std::string files_path =
 347       reproducer_root.CopyByAppendingPathComponent("files.txt").GetPath();
 348   std::string dirs_path =
 349       reproducer_root.CopyByAppendingPathComponent("dirs.txt").GetPath();
 350
 351   FileCollector collector(
 352       reproducer_root.CopyByAppendingPathComponent("root").GetPath(),
 353       reproducer_root.GetPath());
 354
 355   if (Error e =
 356           addPaths(files_path, [&](StringRef p) { collector.addFile(p); }))
 357     return e;
 358
 359   if (Error e =
 360           addPaths(dirs_path, [&](StringRef p) { collector.addDirectory(p); }))
 361     return e;
 362
 363   FileSpec mapping =
 364       reproducer_root.CopyByAppendingPathComponent(FileProvider::Info::file);
 365   if (auto ec = collector.copyFiles(/*StopOnError=*/false))
 366     return errorCodeToError(ec);
 367   collector.writeMapping(mapping.GetPath());
 368
 369   return llvm::Error::success();
 370 }
 371
 372 llvm::Error repro::Finalize(const FileSpec &root) {
 373   Loader loader(root);
 374   if (Error e = loader.LoadIndex())
 375     return e;
 376   return Finalize(&loader);
 377 }