chrome/browser/resources/hotword/nacl_manager.js

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 cr.define('hotword', function() {
   6 'use strict';
   7
   8 /**
   9  * Class used to manage the state of the NaCl recognizer plugin. Handles all
  10  * control of the NaCl plugin, including creation, start, stop, trigger, and
  11  * shutdown.
  12  *
  13  * @param {boolean} loggingEnabled Whether audio logging is enabled.
  14  * @param {boolean} hotwordStream Whether the audio input stream is from a
  15  *     hotword stream.
  16  * @constructor
  17  * @extends {cr.EventTarget}
  18  */
  19 function NaClManager(loggingEnabled, hotwordStream) {
  20   /**
  21    * Current state of this manager.
  22    * @private {hotword.NaClManager.ManagerState_}
  23    */
  24   this.recognizerState_ = ManagerState_.UNINITIALIZED;
  25
  26   /**
  27    * The window.timeout ID associated with a pending message.
  28    * @private {?number}
  29    */
  30   this.naclTimeoutId_ = null;
  31
  32   /**
  33    * The expected message that will cancel the current timeout.
  34    * @private {?string}
  35    */
  36   this.expectingMessage_ = null;
  37
  38   /**
  39    * Whether the plugin will be started as soon as it stops.
  40    * @private {boolean}
  41    */
  42   this.restartOnStop_ = false;
  43
  44   /**
  45    * NaCl plugin element on extension background page.
  46    * @private {?HTMLEmbedElement}
  47    */
  48   this.plugin_ = null;
  49
  50   /**
  51    * URL containing hotword-model data file.
  52    * @private {string}
  53    */
  54   this.modelUrl_ = '';
  55
  56   /**
  57    * Media stream containing an audio input track.
  58    * @private {?MediaStream}
  59    */
  60   this.stream_ = null;
  61
  62   /**
  63    * The mode to start the recognizer in.
  64    * @private {?chrome.hotwordPrivate.RecognizerStartMode}
  65    */
  66   this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;
  67
  68   /**
  69    * Whether audio logging is enabled.
  70    * @private {boolean}
  71    */
  72   this.loggingEnabled_ = loggingEnabled;
  73
  74   /**
  75    * Whether the audio input stream is from a hotword stream.
  76    * @private {boolean}
  77    */
  78   this.hotwordStream_ = hotwordStream;
  79
  80   /**
  81    * Audio log of X seconds before hotword triggered.
  82    * @private {?Object}
  83    */
  84   this.preambleLog_ = null;
  85 };
  86
  87 /**
  88  * States this manager can be in. Since messages to/from the plugin are
  89  * asynchronous (and potentially queued), it's not possible to know what state
  90  * the plugin is in. However, track a state machine for NaClManager based on
  91  * what messages are sent/received.
  92  * @enum {number}
  93  * @private
  94  */
  95 NaClManager.ManagerState_ = {
  96   UNINITIALIZED: 0,
  97   LOADING: 1,
  98   STOPPING: 2,
  99   STOPPED: 3,
 100   STARTING: 4,
 101   RUNNING: 5,
 102   ERROR: 6,
 103   SHUTDOWN: 7,
 104 };
 105 var ManagerState_ = NaClManager.ManagerState_;
 106 var Error_ = hotword.constants.Error;
 107 var UmaNaClMessageTimeout_ = hotword.constants.UmaNaClMessageTimeout;
 108 var UmaNaClPluginLoadResult_ = hotword.constants.UmaNaClPluginLoadResult;
 109
 110 NaClManager.prototype.__proto__ = cr.EventTarget.prototype;
 111
 112 /**
 113  * Called when an error occurs. Dispatches an event.
 114  * @param {!hotword.constants.Error} error
 115  * @private
 116  */
 117 NaClManager.prototype.handleError_ = function(error) {
 118   var event = new Event(hotword.constants.Event.ERROR);
 119   event.data = error;
 120   this.dispatchEvent(event);
 121 };
 122
 123 /**
 124  * Record the result of loading the NaCl plugin to UMA.
 125  * @param {!hotword.constants.UmaNaClPluginLoadResult} error
 126  * @private
 127  */
 128 NaClManager.prototype.logPluginLoadResult_ = function(error) {
 129   hotword.metrics.recordEnum(
 130       hotword.constants.UmaMetrics.NACL_PLUGIN_LOAD_RESULT,
 131       error,
 132       UmaNaClPluginLoadResult_.MAX);
 133 };
 134
 135 /**
 136  * Set a timeout. Only allow one timeout to exist at any given time.
 137  * @param {!function()} func
 138  * @param {number} timeout
 139  * @private
 140  */
 141 NaClManager.prototype.setTimeout_ = function(func, timeout) {
 142   assert(!this.naclTimeoutId_, 'Timeout already exists');
 143   this.naclTimeoutId_ = window.setTimeout(
 144       function() {
 145         this.naclTimeoutId_ = null;
 146         func();
 147       }.bind(this), timeout);
 148 };
 149
 150 /**
 151  * Clears the current timeout.
 152  * @private
 153  */
 154 NaClManager.prototype.clearTimeout_ = function() {
 155   window.clearTimeout(this.naclTimeoutId_);
 156   this.naclTimeoutId_ = null;
 157 };
 158
 159 /**
 160  * Starts a stopped or stopping hotword recognizer (NaCl plugin).
 161  * @param {hotword.constants.RecognizerStartMode} mode The mode to start the
 162  *     recognizer in.
 163  */
 164 NaClManager.prototype.startRecognizer = function(mode) {
 165   this.startMode_ = mode;
 166   if (this.recognizerState_ == ManagerState_.STOPPED) {
 167     this.preambleLog_ = null;
 168     this.recognizerState_ = ManagerState_.STARTING;
 169     if (mode == hotword.constants.RecognizerStartMode.NEW_MODEL) {
 170       hotword.debug('Starting Recognizer in START training mode');
 171       this.sendDataToPlugin_(hotword.constants.NaClPlugin.BEGIN_SPEAKER_MODEL);
 172     } else if (mode == hotword.constants.RecognizerStartMode.ADAPT_MODEL) {
 173       hotword.debug('Starting Recognizer in ADAPT training mode');
 174       this.sendDataToPlugin_(hotword.constants.NaClPlugin.ADAPT_SPEAKER_MODEL);
 175     } else {
 176       hotword.debug('Starting Recognizer in NORMAL mode');
 177       this.sendDataToPlugin_(hotword.constants.NaClPlugin.RESTART);
 178     }
 179     // Normally, there would be a waitForMessage_(READY_FOR_AUDIO) here.
 180     // However, this message is sent the first time audio data is read and in
 181     // some cases (ie. using the hotword stream), this won't happen until a
 182     // potential hotword trigger is seen. Having a waitForMessage_() would time
 183     // out in this case, so just leave it out. This ends up sacrificing a bit of
 184     // error detection in the non-hotword-stream case, but I think we can live
 185     // with that.
 186   } else if (this.recognizerState_ == ManagerState_.STOPPING) {
 187     // Wait until the plugin is stopped before trying to start it.
 188     this.restartOnStop_ = true;
 189   } else {
 190     throw 'Attempting to start NaCl recogniser not in STOPPED or STOPPING ' +
 191         'state';
 192   }
 193 };
 194
 195 /**
 196  * Stops the hotword recognizer.
 197  */
 198 NaClManager.prototype.stopRecognizer = function() {
 199   if (this.recognizerState_ == ManagerState_.STARTING) {
 200     // If the recognizer is stopped before it finishes starting, it causes an
 201     // assertion to be raised in waitForMessage_() since we're waiting for the
 202     // READY_FOR_AUDIO message. Clear the current timeout and expecting message
 203     // since we no longer expect it and may never receive it.
 204     this.clearTimeout_();
 205     this.expectingMessage_ = null;
 206   }
 207   this.sendDataToPlugin_(hotword.constants.NaClPlugin.STOP);
 208   this.recognizerState_ = ManagerState_.STOPPING;
 209   this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
 210                        hotword.constants.NaClPlugin.STOPPED);
 211 };
 212
 213 /**
 214  * Saves the speaker model.
 215  */
 216 NaClManager.prototype.finalizeSpeakerModel = function() {
 217   if (this.recognizerState_ == ManagerState_.UNINITIALIZED ||
 218       this.recognizerState_ == ManagerState_.ERROR ||
 219       this.recognizerState_ == ManagerState_.SHUTDOWN ||
 220       this.recognizerState_ == ManagerState_.LOADING) {
 221     return;
 222   }
 223   this.sendDataToPlugin_(hotword.constants.NaClPlugin.FINISH_SPEAKER_MODEL);
 224 };
 225
 226 /**
 227  * Checks whether the file at the given path exists.
 228  * @param {!string} path Path to a file. Can be any valid URL.
 229  * @return {boolean} True if the patch exists.
 230  * @private
 231  */
 232 NaClManager.prototype.fileExists_ = function(path) {
 233   var xhr = new XMLHttpRequest();
 234   xhr.open('HEAD', path, false);
 235   try {
 236     xhr.send();
 237   } catch (err) {
 238     return false;
 239   }
 240   if (xhr.readyState != xhr.DONE || xhr.status != 200) {
 241     return false;
 242   }
 243   return true;
 244 };
 245
 246 /**
 247  * Creates and returns a list of possible languages to check for hotword
 248  * support.
 249  * @return {!Array<string>} Array of languages.
 250  * @private
 251  */
 252 NaClManager.prototype.getPossibleLanguages_ = function() {
 253   // Create array used to search first for language-country, if not found then
 254   // search for language, if not found then no language (empty string).
 255   // For example, search for 'en-us', then 'en', then ''.
 256   var langs = new Array();
 257   if (hotword.constants.UI_LANGUAGE) {
 258     // Chrome webstore doesn't support uppercase path: crbug.com/353407
 259     var language = hotword.constants.UI_LANGUAGE.toLowerCase();
 260     langs.push(language);  // Example: 'en-us'.
 261     // Remove country to add just the language to array.
 262     var hyphen = language.lastIndexOf('-');
 263     if (hyphen >= 0) {
 264       langs.push(language.substr(0, hyphen));  // Example: 'en'.
 265     }
 266   }
 267   langs.push('');
 268   return langs;
 269 };
 270
 271 /**
 272  * Creates a NaCl plugin object and attaches it to the page.
 273  * @param {!string} src Location of the plugin.
 274  * @return {!HTMLEmbedElement} NaCl plugin DOM object.
 275  * @private
 276  */
 277 NaClManager.prototype.createPlugin_ = function(src) {
 278   var plugin = /** @type {HTMLEmbedElement} */(document.createElement('embed'));
 279   plugin.src = src;
 280   plugin.type = 'application/x-nacl';
 281   document.body.appendChild(plugin);
 282   return plugin;
 283 };
 284
 285 /**
 286  * Initializes the NaCl manager.
 287  * @param {!string} naclArch Either 'arm', 'x86-32' or 'x86-64'.
 288  * @param {!MediaStream} stream A stream containing an audio source track.
 289  * @return {boolean} True if the successful.
 290  */
 291 NaClManager.prototype.initialize = function(naclArch, stream) {
 292   assert(this.recognizerState_ == ManagerState_.UNINITIALIZED,
 293          'Recognizer not in uninitialized state. State: ' +
 294          this.recognizerState_);
 295   assert(this.plugin_ == null);
 296   var langs = this.getPossibleLanguages_();
 297   var i, j;
 298   // For country-lang variations. For example, when combined with path it will
 299   // attempt to find: '/x86-32_en-gb/', else '/x86-32_en/', else '/x86-32_/'.
 300   for (i = 0; i < langs.length; i++) {
 301     var folder = hotword.constants.SHARED_MODULE_ROOT + '/_platform_specific/' +
 302         naclArch + '_' + langs[i] + '/';
 303     var dataSrc = folder + hotword.constants.File.RECOGNIZER_CONFIG;
 304     var pluginSrc = hotword.constants.SHARED_MODULE_ROOT + '/hotword_' +
 305         langs[i] + '.nmf';
 306     var dataExists = this.fileExists_(dataSrc) && this.fileExists_(pluginSrc);
 307     if (!dataExists) {
 308       continue;
 309     }
 310
 311     var plugin = this.createPlugin_(pluginSrc);
 312     if (!plugin || !plugin.postMessage) {
 313       document.body.removeChild(plugin);
 314       this.recognizerState_ = ManagerState_.ERROR;
 315       return false;
 316     }
 317     this.plugin_ = plugin;
 318     this.modelUrl_ = chrome.extension.getURL(dataSrc);
 319     this.stream_ = stream;
 320     this.recognizerState_ = ManagerState_.LOADING;
 321
 322     plugin.addEventListener('message',
 323                             this.handlePluginMessage_.bind(this),
 324                             false);
 325
 326     plugin.addEventListener('crash',
 327                             function() {
 328                               this.handleError_(Error_.NACL_CRASH);
 329                               this.logPluginLoadResult_(
 330                                   UmaNaClPluginLoadResult_.CRASH);
 331                             }.bind(this),
 332                             false);
 333     return true;
 334   }
 335   this.recognizerState_ = ManagerState_.ERROR;
 336   this.logPluginLoadResult_(UmaNaClPluginLoadResult_.NO_MODULE_FOUND);
 337   return false;
 338 };
 339
 340 /**
 341  * Shuts down the NaCl plugin and frees all resources.
 342  */
 343 NaClManager.prototype.shutdown = function() {
 344   if (this.plugin_ != null) {
 345     document.body.removeChild(this.plugin_);
 346     this.plugin_ = null;
 347   }
 348   this.clearTimeout_();
 349   this.recognizerState_ = ManagerState_.SHUTDOWN;
 350   if (this.stream_)
 351     this.stream_.stop();
 352   this.stream_ = null;
 353 };
 354
 355 /**
 356  * Sends data to the NaCl plugin.
 357  * @param {!string|!MediaStreamTrack} data Command to be sent to NaCl plugin.
 358  * @private
 359  */
 360 NaClManager.prototype.sendDataToPlugin_ = function(data) {
 361   assert(this.recognizerState_ != ManagerState_.UNINITIALIZED,
 362          'Recognizer in uninitialized state');
 363   this.plugin_.postMessage(data);
 364 };
 365
 366 /**
 367  * Waits, with a timeout, for a message to be received from the plugin. If the
 368  * message is not seen within the timeout, dispatch an 'error' event and go into
 369  * the ERROR state.
 370  * @param {number} timeout Timeout, in milliseconds, to wait for the message.
 371  * @param {!string} message Message to wait for.
 372  * @private
 373  */
 374 NaClManager.prototype.waitForMessage_ = function(timeout, message) {
 375   assert(this.expectingMessage_ == null, 'Cannot wait for message: ' +
 376       message + ', already waiting for message ' + this.expectingMessage_);
 377   this.setTimeout_(
 378       function() {
 379         this.recognizerState_ = ManagerState_.ERROR;
 380         this.handleError_(Error_.TIMEOUT);
 381         switch (this.expectingMessage_) {
 382           case hotword.constants.NaClPlugin.REQUEST_MODEL:
 383             var metricValue = UmaNaClMessageTimeout_.REQUEST_MODEL;
 384             break;
 385           case hotword.constants.NaClPlugin.MODEL_LOADED:
 386             var metricValue = UmaNaClMessageTimeout_.MODEL_LOADED;
 387             break;
 388           case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
 389             var metricValue = UmaNaClMessageTimeout_.READY_FOR_AUDIO;
 390             break;
 391           case hotword.constants.NaClPlugin.STOPPED:
 392             var metricValue = UmaNaClMessageTimeout_.STOPPED;
 393             break;
 394           case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
 395             var metricValue = UmaNaClMessageTimeout_.HOTWORD_DETECTED;
 396             break;
 397           case hotword.constants.NaClPlugin.MS_CONFIGURED:
 398             var metricValue = UmaNaClMessageTimeout_.MS_CONFIGURED;
 399             break;
 400         }
 401         hotword.metrics.recordEnum(
 402             hotword.constants.UmaMetrics.NACL_MESSAGE_TIMEOUT,
 403             metricValue,
 404             UmaNaClMessageTimeout_.MAX);
 405       }.bind(this), timeout);
 406   this.expectingMessage_ = message;
 407 };
 408
 409 /**
 410  * Called when a message is received from the plugin. If we're waiting for that
 411  * message, cancel the pending timeout.
 412  * @param {string} message Message received.
 413  * @private
 414  */
 415 NaClManager.prototype.receivedMessage_ = function(message) {
 416   if (message == this.expectingMessage_) {
 417     this.clearTimeout_();
 418     this.expectingMessage_ = null;
 419   }
 420 };
 421
 422 /**
 423  * Handle a REQUEST_MODEL message from the plugin.
 424  * The plugin sends this message immediately after starting.
 425  * @private
 426  */
 427 NaClManager.prototype.handleRequestModel_ = function() {
 428   if (this.recognizerState_ != ManagerState_.LOADING) {
 429     return;
 430   }
 431   this.logPluginLoadResult_(UmaNaClPluginLoadResult_.SUCCESS);
 432   this.sendDataToPlugin_(
 433       hotword.constants.NaClPlugin.MODEL_PREFIX + this.modelUrl_);
 434   this.waitForMessage_(hotword.constants.TimeoutMs.LONG,
 435                        hotword.constants.NaClPlugin.MODEL_LOADED);
 436
 437   // Configure logging in the plugin. This can be configured any time before
 438   // starting the recognizer, and now is as good a time as any.
 439   if (this.loggingEnabled_) {
 440     this.sendDataToPlugin_(
 441         hotword.constants.NaClPlugin.LOG + ':' +
 442         hotword.constants.AUDIO_LOG_SECONDS);
 443   }
 444
 445   // If the audio stream is from a hotword stream, tell the plugin.
 446   if (this.hotwordStream_) {
 447     this.sendDataToPlugin_(
 448         hotword.constants.NaClPlugin.DSP + ':' +
 449         hotword.constants.HOTWORD_STREAM_TIMEOUT_SECONDS);
 450   }
 451 };
 452
 453 /**
 454  * Handle a MODEL_LOADED message from the plugin.
 455  * The plugin sends this message after successfully loading the language model.
 456  * @private
 457  */
 458 NaClManager.prototype.handleModelLoaded_ = function() {
 459   if (this.recognizerState_ != ManagerState_.LOADING) {
 460     return;
 461   }
 462   this.sendDataToPlugin_(this.stream_.getAudioTracks()[0]);
 463   this.waitForMessage_(hotword.constants.TimeoutMs.LONG,
 464                        hotword.constants.NaClPlugin.MS_CONFIGURED);
 465 };
 466
 467 /**
 468  * Handle a MS_CONFIGURED message from the plugin.
 469  * The plugin sends this message after successfully configuring the audio input
 470  * stream.
 471  * @private
 472  */
 473 NaClManager.prototype.handleMsConfigured_ = function() {
 474   if (this.recognizerState_ != ManagerState_.LOADING) {
 475     return;
 476   }
 477   this.recognizerState_ = ManagerState_.STOPPED;
 478   this.dispatchEvent(new Event(hotword.constants.Event.READY));
 479 };
 480
 481 /**
 482  * Handle a READY_FOR_AUDIO message from the plugin.
 483  * The plugin sends this message after the recognizer is started and
 484  * successfully receives and processes audio data.
 485  * @private
 486  */
 487 NaClManager.prototype.handleReadyForAudio_ = function() {
 488   if (this.recognizerState_ != ManagerState_.STARTING) {
 489     return;
 490   }
 491   this.recognizerState_ = ManagerState_.RUNNING;
 492 };
 493
 494 /**
 495  * Handle a HOTWORD_DETECTED message from the plugin.
 496  * The plugin sends this message after detecting the hotword.
 497  * @private
 498  */
 499 NaClManager.prototype.handleHotwordDetected_ = function() {
 500   if (this.recognizerState_ != ManagerState_.RUNNING) {
 501     return;
 502   }
 503   // We'll receive a STOPPED message very soon.
 504   this.recognizerState_ = ManagerState_.STOPPING;
 505   this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
 506                        hotword.constants.NaClPlugin.STOPPED);
 507   var event = new Event(hotword.constants.Event.TRIGGER);
 508   event.log = this.preambleLog_;
 509   this.dispatchEvent(event);
 510 };
 511
 512 /**
 513  * Handle a STOPPED message from the plugin.
 514  * This plugin sends this message after stopping the recognizer. This can happen
 515  * either in response to a stop request, or after the hotword is detected.
 516  * @private
 517  */
 518 NaClManager.prototype.handleStopped_ = function() {
 519   this.recognizerState_ = ManagerState_.STOPPED;
 520   if (this.restartOnStop_) {
 521     this.restartOnStop_ = false;
 522     this.startRecognizer(this.startMode_);
 523   }
 524 };
 525
 526 /**
 527  * Handle a TIMEOUT message from the plugin.
 528  * The plugin sends this message when it thinks the stream is from a DSP and
 529  * a hotword wasn't detected within a timeout period after arrival of the first
 530  * audio samples.
 531  * @private
 532  */
 533 NaClManager.prototype.handleTimeout_ = function() {
 534   if (this.recognizerState_ != ManagerState_.RUNNING) {
 535     return;
 536   }
 537   this.recognizerState_ = ManagerState_.STOPPED;
 538   this.dispatchEvent(new Event(hotword.constants.Event.TIMEOUT));
 539 };
 540
 541 /**
 542  * Handle a SPEAKER_MODEL_SAVED message from the plugin.
 543  * The plugin sends this message after writing the model to a file.
 544  * @private
 545  */
 546 NaClManager.prototype.handleSpeakerModelSaved_ = function() {
 547   this.dispatchEvent(new Event(hotword.constants.Event.SPEAKER_MODEL_SAVED));
 548 };
 549
 550 /**
 551  * Handles a message from the NaCl plugin.
 552  * @param {!Event} msg Message from NaCl plugin.
 553  * @private
 554  */
 555 NaClManager.prototype.handlePluginMessage_ = function(msg) {
 556   if (msg['data']) {
 557     if (typeof(msg['data']) == 'object') {
 558       // Save the preamble for delivery to the trigger handler when the trigger
 559       // message arrives.
 560       this.preambleLog_ = msg['data'];
 561       return;
 562     }
 563     this.receivedMessage_(msg['data']);
 564     switch (msg['data']) {
 565       case hotword.constants.NaClPlugin.REQUEST_MODEL:
 566         this.handleRequestModel_();
 567         break;
 568       case hotword.constants.NaClPlugin.MODEL_LOADED:
 569         this.handleModelLoaded_();
 570         break;
 571       case hotword.constants.NaClPlugin.MS_CONFIGURED:
 572         this.handleMsConfigured_();
 573         break;
 574       case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
 575         this.handleReadyForAudio_();
 576         break;
 577       case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
 578         this.handleHotwordDetected_();
 579         break;
 580       case hotword.constants.NaClPlugin.STOPPED:
 581         this.handleStopped_();
 582         break;
 583       case hotword.constants.NaClPlugin.TIMEOUT:
 584         this.handleTimeout_();
 585         break;
 586       case hotword.constants.NaClPlugin.SPEAKER_MODEL_SAVED:
 587         this.handleSpeakerModelSaved_();
 588         break;
 589     }
 590   }
 591 };
 592
 593 return {
 594   NaClManager: NaClManager
 595 };
 596
 597 });