tools/gn/input_file_manager.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "tools/gn/input_file_manager.h"
   6
   7 #include "base/bind.h"
   8 #include "base/stl_util.h"
   9 #include "tools/gn/filesystem_utils.h"
  10 #include "tools/gn/parser.h"
  11 #include "tools/gn/scheduler.h"
  12 #include "tools/gn/scope_per_file_provider.h"
  13 #include "tools/gn/tokenizer.h"
  14 #include "tools/gn/trace.h"
  15
  16 namespace {
  17
  18 void InvokeFileLoadCallback(const InputFileManager::FileLoadCallback& cb,
  19                             const ParseNode* node) {
  20   cb.Run(node);
  21 }
  22
  23 bool DoLoadFile(const LocationRange& origin,
  24                 const BuildSettings* build_settings,
  25                 const SourceFile& name,
  26                 InputFile* file,
  27                 std::vector<Token>* tokens,
  28                 scoped_ptr<ParseNode>* root,
  29                 Err* err) {
  30   // Do all of this stuff outside the lock. We should not give out file
  31   // pointers until the read is complete.
  32   if (g_scheduler->verbose_logging()) {
  33     std::string logmsg = name.value();
  34     if (origin.begin().file())
  35       logmsg += " (referenced from " + origin.begin().Describe(false) + ")";
  36     g_scheduler->Log("Loading", logmsg);
  37   }
  38
  39   // Read.
  40   base::FilePath primary_path = build_settings->GetFullPath(name);
  41   ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, name.value());
  42   if (!file->Load(primary_path)) {
  43     if (!build_settings->secondary_source_path().empty()) {
  44       // Fall back to secondary source tree.
  45       base::FilePath secondary_path =
  46           build_settings->GetFullPathSecondary(name);
  47       if (!file->Load(secondary_path)) {
  48         *err = Err(origin, "Can't load input file.",
  49                    "Unable to load:\n  " +
  50                    FilePathToUTF8(primary_path) + "\n"
  51                    "I also checked in the secondary tree for:\n  " +
  52                    FilePathToUTF8(secondary_path));
  53         return false;
  54       }
  55     } else {
  56       *err = Err(origin,
  57                  "Unable to load \"" + FilePathToUTF8(primary_path) + "\".");
  58       return false;
  59     }
  60   }
  61   load_trace.Done();
  62
  63   ScopedTrace exec_trace(TraceItem::TRACE_FILE_PARSE, name.value());
  64
  65   // Tokenize.
  66   *tokens = Tokenizer::Tokenize(file, err);
  67   if (err->has_error())
  68     return false;
  69
  70   // Parse.
  71   *root = Parser::Parse(*tokens, err);
  72   if (err->has_error())
  73     return false;
  74
  75   exec_trace.Done();
  76   return true;
  77 }
  78
  79 }  // namespace
  80
  81 InputFileManager::InputFileData::InputFileData(const SourceFile& file_name)
  82     : file(file_name),
  83       loaded(false),
  84       sync_invocation(false) {
  85 }
  86
  87 InputFileManager::InputFileData::~InputFileData() {
  88 }
  89
  90 InputFileManager::InputFileManager() {
  91 }
  92
  93 InputFileManager::~InputFileManager() {
  94   // Should be single-threaded by now.
  95   STLDeleteContainerPairSecondPointers(input_files_.begin(),
  96                                        input_files_.end());
  97   STLDeleteContainerPointers(dynamic_inputs_.begin(), dynamic_inputs_.end());
  98 }
  99
 100 bool InputFileManager::AsyncLoadFile(const LocationRange& origin,
 101                                      const BuildSettings* build_settings,
 102                                      const SourceFile& file_name,
 103                                      const FileLoadCallback& callback,
 104                                      Err* err) {
 105   // Try not to schedule callbacks while holding the lock. All cases that don't
 106   // want to schedule should return early. Otherwise, this will be scheduled
 107   // after we leave the lock.
 108   base::Closure schedule_this;
 109   {
 110     base::AutoLock lock(lock_);
 111
 112     InputFileMap::const_iterator found = input_files_.find(file_name);
 113     if (found == input_files_.end()) {
 114       // New file, schedule load.
 115       InputFileData* data = new InputFileData(file_name);
 116       data->scheduled_callbacks.push_back(callback);
 117       input_files_[file_name] = data;
 118
 119       schedule_this = base::Bind(&InputFileManager::BackgroundLoadFile,
 120                                  this,
 121                                  origin,
 122                                  build_settings,
 123                                  file_name,
 124                                  &data->file);
 125     } else {
 126       InputFileData* data = found->second;
 127
 128       // Prevent mixing async and sync loads. See SyncLoadFile for discussion.
 129       if (data->sync_invocation) {
 130         g_scheduler->FailWithError(Err(
 131             origin, "Load type mismatch.",
 132             "The file \"" + file_name.value() + "\" was previously loaded\n"
 133             "synchronously (via an import) and now you're trying to load it "
 134             "asynchronously\n(via a deps rule). This is a class 2 misdemeanor: "
 135             "a single input file must\nbe loaded the same way each time to "
 136             "avoid blowing my tiny, tiny mind."));
 137         return false;
 138       }
 139
 140       if (data->loaded) {
 141         // Can just directly issue the callback on the background thread.
 142         schedule_this = base::Bind(&InvokeFileLoadCallback, callback,
 143                                    data->parsed_root.get());
 144       } else {
 145         // Load is pending on this file, schedule the invoke.
 146         data->scheduled_callbacks.push_back(callback);
 147         return true;
 148       }
 149     }
 150   }
 151   g_scheduler->pool()->PostWorkerTaskWithShutdownBehavior(
 152       FROM_HERE, schedule_this,
 153       base::SequencedWorkerPool::BLOCK_SHUTDOWN);
 154   return true;
 155 }
 156
 157 const ParseNode* InputFileManager::SyncLoadFile(
 158     const LocationRange& origin,
 159     const BuildSettings* build_settings,
 160     const SourceFile& file_name,
 161     Err* err) {
 162   base::AutoLock lock(lock_);
 163
 164   InputFileData* data = nullptr;
 165   InputFileMap::iterator found = input_files_.find(file_name);
 166   if (found == input_files_.end()) {
 167     // Haven't seen this file yet, start loading right now.
 168     data = new InputFileData(file_name);
 169     data->sync_invocation = true;
 170     input_files_[file_name] = data;
 171
 172     base::AutoUnlock unlock(lock_);
 173     if (!LoadFile(origin, build_settings, file_name, &data->file, err))
 174       return nullptr;
 175   } else {
 176     // This file has either been loaded or is pending loading.
 177     data = found->second;
 178
 179     if (!data->sync_invocation) {
 180       // Don't allow mixing of sync and async loads. If an async load is
 181       // scheduled and then a bunch of threads need to load it synchronously
 182       // and block on it loading, it could deadlock or at least cause a lot
 183       // of wasted CPU while those threads wait for the load to complete (which
 184       // may be far back in the input queue).
 185       //
 186       // We could work around this by promoting the load to a sync load. This
 187       // requires a bunch of extra code to either check flags and likely do
 188       // extra locking (bad) or to just do both types of load on the file and
 189       // deal with the race condition.
 190       //
 191       // I have no practical way to test this, and generally we should have
 192       // all include files processed synchronously and all build files
 193       // processed asynchronously, so it doesn't happen in practice.
 194       *err = Err(
 195           origin, "Load type mismatch.",
 196           "The file \"" + file_name.value() + "\" was previously loaded\n"
 197           "asynchronously (via a deps rule) and now you're trying to load it "
 198           "synchronously.\nThis is a class 2 misdemeanor: a single input file "
 199           "must be loaded the same way\neach time to avoid blowing my tiny, "
 200           "tiny mind.");
 201       return nullptr;
 202     }
 203
 204     if (!data->loaded) {
 205       // Wait for the already-pending sync load to complete.
 206       if (!data->completion_event)
 207         data->completion_event.reset(new base::WaitableEvent(false, false));
 208       {
 209         base::AutoUnlock unlock(lock_);
 210         data->completion_event->Wait();
 211       }
 212       // If there were multiple waiters on the same event, we now need to wake
 213       // up the next one.
 214       data->completion_event->Signal();
 215     }
 216   }
 217
 218   // The other load could have failed. It is possible that this thread's error
 219   // will be reported to the scheduler before the other thread's (and the first
 220   // error reported "wins"). Forward the parse error from the other load for
 221   // this thread so that the error message is useful.
 222   if (!data->parsed_root)
 223     *err = data->parse_error;
 224   return data->parsed_root.get();
 225 }
 226
 227 void InputFileManager::AddDynamicInput(const SourceFile& name,
 228                                        InputFile** file,
 229                                        std::vector<Token>** tokens,
 230                                        scoped_ptr<ParseNode>** parse_root) {
 231   InputFileData* data = new InputFileData(name);
 232   {
 233     base::AutoLock lock(lock_);
 234     dynamic_inputs_.push_back(data);
 235   }
 236   *file = &data->file;
 237   *tokens = &data->tokens;
 238   *parse_root = &data->parsed_root;
 239 }
 240
 241 int InputFileManager::GetInputFileCount() const {
 242   base::AutoLock lock(lock_);
 243   return static_cast<int>(input_files_.size());
 244 }
 245
 246 void InputFileManager::GetAllPhysicalInputFileNames(
 247     std::vector<base::FilePath>* result) const {
 248   base::AutoLock lock(lock_);
 249   result->reserve(input_files_.size());
 250   for (const auto& file : input_files_) {
 251     if (!file.second->file.physical_name().empty())
 252       result->push_back(file.second->file.physical_name());
 253   }
 254 }
 255
 256 void InputFileManager::BackgroundLoadFile(const LocationRange& origin,
 257                                           const BuildSettings* build_settings,
 258                                           const SourceFile& name,
 259                                           InputFile* file) {
 260   Err err;
 261   if (!LoadFile(origin, build_settings, name, file, &err))
 262     g_scheduler->FailWithError(err);
 263 }
 264
 265 bool InputFileManager::LoadFile(const LocationRange& origin,
 266                                 const BuildSettings* build_settings,
 267                                 const SourceFile& name,
 268                                 InputFile* file,
 269                                 Err* err) {
 270   std::vector<Token> tokens;
 271   scoped_ptr<ParseNode> root;
 272   bool success = DoLoadFile(origin, build_settings, name, file,
 273                             &tokens, &root, err);
 274   // Can't return early. We have to ensure that the completion event is
 275   // signaled in all cases bacause another thread could be blocked on this one.
 276
 277   // Save this pointer for running the callbacks below, which happens after the
 278   // scoped ptr ownership is taken away inside the lock.
 279   ParseNode* unowned_root = root.get();
 280
 281   std::vector<FileLoadCallback> callbacks;
 282   {
 283     base::AutoLock lock(lock_);
 284     DCHECK(input_files_.find(name) != input_files_.end());
 285
 286     InputFileData* data = input_files_[name];
 287     data->loaded = true;
 288     if (success) {
 289       data->tokens.swap(tokens);
 290       data->parsed_root = root.Pass();
 291     } else {
 292       data->parse_error = *err;
 293     }
 294
 295     // Unblock waiters on this event.
 296     //
 297     // It's somewhat bad to signal this inside the lock. When it's used, it's
 298     // lazily created inside the lock. So we need to do the check and signal
 299     // inside the lock to avoid race conditions on the lazy creation of the
 300     // lock.
 301     //
 302     // We could avoid this by creating the lock every time, but the lock is
 303     // very seldom used and will generally be NULL, so my current theory is that
 304     // several signals of a completion event inside a lock is better than
 305     // creating about 1000 extra locks (one for each file).
 306     if (data->completion_event)
 307       data->completion_event->Signal();
 308
 309     callbacks.swap(data->scheduled_callbacks);
 310   }
 311
 312   // Run pending invocations. Theoretically we could schedule each of these
 313   // separately to get some parallelism. But normally there will only be one
 314   // item in the list, so that's extra overhead and complexity for no gain.
 315   if (success) {
 316     for (const auto& cb : callbacks)
 317       cb.Run(unowned_root);
 318   }
 319   return success;
 320 }