tools/gn/input_file_manager.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "tools/gn/input_file_manager.h"
   6
   7 #include "base/bind.h"
   8 #include "base/stl_util.h"
   9 #include "tools/gn/filesystem_utils.h"
  10 #include "tools/gn/parser.h"
  11 #include "tools/gn/scheduler.h"
  12 #include "tools/gn/scope_per_file_provider.h"
  13 #include "tools/gn/tokenizer.h"
  14 #include "tools/gn/trace.h"
  15
  16 namespace {
  17
  18 void InvokeFileLoadCallback(const InputFileManager::FileLoadCallback& cb,
  19                             const ParseNode* node) {
  20   cb.Run(node);
  21 }
  22
  23 }  // namespace
  24
  25 InputFileManager::InputFileData::InputFileData(const SourceFile& file_name)
  26     : file(file_name),
  27       loaded(false),
  28       sync_invocation(false) {
  29 }
  30
  31 InputFileManager::InputFileData::~InputFileData() {
  32 }
  33
  34 InputFileManager::InputFileManager() {
  35 }
  36
  37 InputFileManager::~InputFileManager() {
  38   // Should be single-threaded by now.
  39   STLDeleteContainerPairSecondPointers(input_files_.begin(),
  40                                        input_files_.end());
  41   STLDeleteContainerPointers(dynamic_inputs_.begin(), dynamic_inputs_.end());
  42 }
  43
  44 bool InputFileManager::AsyncLoadFile(const LocationRange& origin,
  45                                      const BuildSettings* build_settings,
  46                                      const SourceFile& file_name,
  47                                      const FileLoadCallback& callback,
  48                                      Err* err) {
  49   // Try not to schedule callbacks while holding the lock. All cases that don't
  50   // want to schedule should return early. Otherwise, this will be scheduled
  51   // after we leave the lock.
  52   base::Closure schedule_this;
  53   {
  54     base::AutoLock lock(lock_);
  55
  56     InputFileMap::const_iterator found = input_files_.find(file_name);
  57     if (found == input_files_.end()) {
  58       // New file, schedule load.
  59       InputFileData* data = new InputFileData(file_name);
  60       data->scheduled_callbacks.push_back(callback);
  61       input_files_[file_name] = data;
  62
  63       schedule_this = base::Bind(&InputFileManager::BackgroundLoadFile,
  64                                  this,
  65                                  origin,
  66                                  build_settings,
  67                                  file_name,
  68                                  &data->file);
  69     } else {
  70       InputFileData* data = found->second;
  71
  72       // Prevent mixing async and sync loads. See SyncLoadFile for discussion.
  73       if (data->sync_invocation) {
  74         g_scheduler->FailWithError(Err(
  75             origin, "Load type mismatch.",
  76             "The file \"" + file_name.value() + "\" was previously loaded\n"
  77             "synchronously (via an import) and now you're trying to load it "
  78             "asynchronously\n(via a deps rule). This is a class 2 misdemeanor: "
  79             "a single input file must\nbe loaded the same way each time to "
  80             "avoid blowing my tiny, tiny mind."));
  81         return false;
  82       }
  83
  84       if (data->loaded) {
  85         // Can just directly issue the callback on the background thread.
  86         schedule_this = base::Bind(&InvokeFileLoadCallback, callback,
  87                                    data->parsed_root.get());
  88       } else {
  89         // Load is pending on this file, schedule the invoke.
  90         data->scheduled_callbacks.push_back(callback);
  91         return true;
  92       }
  93     }
  94   }
  95   g_scheduler->pool()->PostWorkerTaskWithShutdownBehavior(
  96       FROM_HERE, schedule_this,
  97       base::SequencedWorkerPool::BLOCK_SHUTDOWN);
  98   return true;
  99 }
 100
 101 const ParseNode* InputFileManager::SyncLoadFile(
 102     const LocationRange& origin,
 103     const BuildSettings* build_settings,
 104     const SourceFile& file_name,
 105     Err* err) {
 106   base::AutoLock lock(lock_);
 107
 108   InputFileData* data = NULL;
 109   InputFileMap::iterator found = input_files_.find(file_name);
 110   if (found == input_files_.end()) {
 111     // Haven't seen this file yet, start loading right now.
 112     data = new InputFileData(file_name);
 113     data->sync_invocation = true;
 114     input_files_[file_name] = data;
 115
 116     base::AutoUnlock unlock(lock_);
 117     if (!LoadFile(origin, build_settings, file_name, &data->file, err))
 118       return NULL;
 119   } else {
 120     // This file has either been loaded or is pending loading.
 121     data = found->second;
 122
 123     if (!data->sync_invocation) {
 124       // Don't allow mixing of sync and async loads. If an async load is
 125       // scheduled and then a bunch of threads need to load it synchronously
 126       // and block on it loading, it could deadlock or at least cause a lot
 127       // of wasted CPU while those threads wait for the load to complete (which
 128       // may be far back in the input queue).
 129       //
 130       // We could work around this by promoting the load to a sync load. This
 131       // requires a bunch of extra code to either check flags and likely do
 132       // extra locking (bad) or to just do both types of load on the file and
 133       // deal with the race condition.
 134       //
 135       // I have no practical way to test this, and generally we should have
 136       // all include files processed synchronously and all build files
 137       // processed asynchronously, so it doesn't happen in practice.
 138       *err = Err(
 139           origin, "Load type mismatch.",
 140           "The file \"" + file_name.value() + "\" was previously loaded\n"
 141           "asynchronously (via a deps rule) and now you're trying to load it "
 142           "synchronously.\nThis is a class 2 misdemeanor: a single input file "
 143           "must be loaded the same way\neach time to avoid blowing my tiny, "
 144           "tiny mind.");
 145       return NULL;
 146     }
 147
 148     if (!data->loaded) {
 149       // Wait for the already-pending sync load to complete.
 150       if (!data->completion_event)
 151         data->completion_event.reset(new base::WaitableEvent(false, false));
 152       {
 153         base::AutoUnlock unlock(lock_);
 154         data->completion_event->Wait();
 155       }
 156       // If there were multiple waiters on the same event, we now need to wake
 157       // up the next one.
 158       data->completion_event->Signal();
 159     }
 160   }
 161
 162   // The other load could have failed. In this case that error will be printed
 163   // to the console, but we need to return something here, so make up a
 164   // dummy error.
 165   if (!data->parsed_root)
 166     *err = Err(origin, "File parse failed");
 167   return data->parsed_root.get();
 168 }
 169
 170 void InputFileManager::AddDynamicInput(const SourceFile& name,
 171                                        InputFile** file,
 172                                        std::vector<Token>** tokens,
 173                                        scoped_ptr<ParseNode>** parse_root) {
 174   InputFileData* data = new InputFileData(name);
 175   {
 176     base::AutoLock lock(lock_);
 177     dynamic_inputs_.push_back(data);
 178   }
 179   *file = &data->file;
 180   *tokens = &data->tokens;
 181   *parse_root = &data->parsed_root;
 182 }
 183
 184 int InputFileManager::GetInputFileCount() const {
 185   base::AutoLock lock(lock_);
 186   return static_cast<int>(input_files_.size());
 187 }
 188
 189 void InputFileManager::GetAllPhysicalInputFileNames(
 190     std::vector<base::FilePath>* result) const {
 191   base::AutoLock lock(lock_);
 192   result->reserve(input_files_.size());
 193   for (InputFileMap::const_iterator i = input_files_.begin();
 194        i != input_files_.end(); ++i) {
 195     if (!i->second->file.physical_name().empty())
 196       result->push_back(i->second->file.physical_name());
 197   }
 198 }
 199
 200 void InputFileManager::BackgroundLoadFile(const LocationRange& origin,
 201                                           const BuildSettings* build_settings,
 202                                           const SourceFile& name,
 203                                           InputFile* file) {
 204   Err err;
 205   if (!LoadFile(origin, build_settings, name, file, &err))
 206     g_scheduler->FailWithError(err);
 207 }
 208
 209 bool InputFileManager::LoadFile(const LocationRange& origin,
 210                                 const BuildSettings* build_settings,
 211                                 const SourceFile& name,
 212                                 InputFile* file,
 213                                 Err* err) {
 214   // Do all of this stuff outside the lock. We should not give out file
 215   // pointers until the read is complete.
 216   if (g_scheduler->verbose_logging()) {
 217     std::string logmsg = name.value();
 218     if (origin.begin().file())
 219       logmsg += " (referenced from " + origin.begin().Describe(false) + ")";
 220     g_scheduler->Log("Loading", logmsg);
 221   }
 222
 223   // Read.
 224   base::FilePath primary_path = build_settings->GetFullPath(name);
 225   ScopedTrace load_trace(TraceItem::TRACE_FILE_LOAD, name.value());
 226   if (!file->Load(primary_path)) {
 227     if (!build_settings->secondary_source_path().empty()) {
 228       // Fall back to secondary source tree.
 229       base::FilePath secondary_path =
 230           build_settings->GetFullPathSecondary(name);
 231       if (!file->Load(secondary_path)) {
 232         *err = Err(origin, "Can't load input file.",
 233                    "Unable to load either \n" +
 234                    FilePathToUTF8(primary_path) + " or \n" +
 235                    FilePathToUTF8(secondary_path));
 236         return false;
 237       }
 238     } else {
 239       *err = Err(origin,
 240                  "Unable to load \"" + FilePathToUTF8(primary_path) + "\".");
 241       return false;
 242     }
 243   }
 244   load_trace.Done();
 245
 246   ScopedTrace exec_trace(TraceItem::TRACE_FILE_PARSE, name.value());
 247
 248   // Tokenize.
 249   std::vector<Token> tokens = Tokenizer::Tokenize(file, err);
 250   if (err->has_error())
 251     return false;
 252
 253   // Parse.
 254   scoped_ptr<ParseNode> root = Parser::Parse(tokens, err);
 255   if (err->has_error())
 256     return false;
 257   ParseNode* unowned_root = root.get();
 258
 259   exec_trace.Done();
 260
 261   std::vector<FileLoadCallback> callbacks;
 262   {
 263     base::AutoLock lock(lock_);
 264     DCHECK(input_files_.find(name) != input_files_.end());
 265
 266     InputFileData* data = input_files_[name];
 267     data->loaded = true;
 268     data->tokens.swap(tokens);
 269     data->parsed_root = root.Pass();
 270
 271     // Unblock waiters on this event.
 272     //
 273     // It's somewhat bad to signal this inside the lock. When it's used, it's
 274     // lazily created inside the lock. So we need to do the check and signal
 275     // inside the lock to avoid race conditions on the lazy creation of the
 276     // lock.
 277     //
 278     // We could avoid this by creating the lock every time, but the lock is
 279     // very seldom used and will generally be NULL, so my current theory is that
 280     // several signals of a completion event inside a lock is better than
 281     // creating about 1000 extra locks (one for each file).
 282     if (data->completion_event)
 283       data->completion_event->Signal();
 284
 285     callbacks.swap(data->scheduled_callbacks);
 286   }
 287
 288   // Run pending invocations. Theoretically we could schedule each of these
 289   // separately to get some parallelism. But normally there will only be one
 290   // item in the list, so that's extra overhead and complexity for no gain.
 291   for (size_t i = 0; i < callbacks.size(); i++)
 292     callbacks[i].Run(unowned_root);
 293   return true;
 294 }