chrome/browser/resources/hotword/nacl_manager.js

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 cr.define('hotword', function() {
   6 'use strict';
   7
   8 /**
   9  * Class used to manage the state of the NaCl recognizer plugin. Handles all
  10  * control of the NaCl plugin, including creation, start, stop, trigger, and
  11  * shutdown.
  12  *
  13  * @param {boolean} loggingEnabled Whether audio logging is enabled.
  14  * @param {boolean} hotwordStream Whether the audio input stream is from a
  15  *     hotword stream.
  16  * @constructor
  17  * @extends {cr.EventTarget}
  18  */
  19 function NaClManager(loggingEnabled, hotwordStream) {
  20   /**
  21    * Current state of this manager.
  22    * @private {hotword.NaClManager.ManagerState_}
  23    */
  24   this.recognizerState_ = ManagerState_.UNINITIALIZED;
  25
  26   /**
  27    * The window.timeout ID associated with a pending message.
  28    * @private {?number}
  29    */
  30   this.naclTimeoutId_ = null;
  31
  32   /**
  33    * The expected message that will cancel the current timeout.
  34    * @private {?string}
  35    */
  36   this.expectingMessage_ = null;
  37
  38   /**
  39    * Whether the plugin will be started as soon as it stops.
  40    * @private {boolean}
  41    */
  42   this.restartOnStop_ = false;
  43
  44   /**
  45    * NaCl plugin element on extension background page.
  46    * @private {?HTMLEmbedElement}
  47    */
  48   this.plugin_ = null;
  49
  50   /**
  51    * URL containing hotword-model data file.
  52    * @private {string}
  53    */
  54   this.modelUrl_ = '';
  55
  56   /**
  57    * Media stream containing an audio input track.
  58    * @private {?MediaStream}
  59    */
  60   this.stream_ = null;
  61
  62   /**
  63    * The mode to start the recognizer in.
  64    * @private {?chrome.hotwordPrivate.RecognizerStartMode}
  65    */
  66   this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;
  67
  68   /**
  69    * Whether audio logging is enabled.
  70    * @private {boolean}
  71    */
  72   this.loggingEnabled_ = loggingEnabled;
  73
  74   /**
  75    * Whether the audio input stream is from a hotword stream.
  76    * @private {boolean}
  77    */
  78   this.hotwordStream_ = hotwordStream;
  79
  80   /**
  81    * Audio log of X seconds before hotword triggered.
  82    * @private {?Object}
  83    */
  84   this.preambleLog_ = null;
  85 };
  86
  87 /**
  88  * States this manager can be in. Since messages to/from the plugin are
  89  * asynchronous (and potentially queued), it's not possible to know what state
  90  * the plugin is in. However, track a state machine for NaClManager based on
  91  * what messages are sent/received.
  92  * @enum {number}
  93  * @private
  94  */
  95 NaClManager.ManagerState_ = {
  96   UNINITIALIZED: 0,
  97   LOADING: 1,
  98   STOPPING: 2,
  99   STOPPED: 3,
 100   STARTING: 4,
 101   RUNNING: 5,
 102   ERROR: 6,
 103   SHUTDOWN: 7,
 104 };
 105 var ManagerState_ = NaClManager.ManagerState_;
 106 var Error_ = hotword.constants.Error;
 107 var UmaNaClMessageTimeout_ = hotword.constants.UmaNaClMessageTimeout;
 108 var UmaNaClPluginLoadResult_ = hotword.constants.UmaNaClPluginLoadResult;
 109
 110 NaClManager.prototype.__proto__ = cr.EventTarget.prototype;
 111
 112 /**
 113  * Called when an error occurs. Dispatches an event.
 114  * @param {!hotword.constants.Error} error
 115  * @private
 116  */
 117 NaClManager.prototype.handleError_ = function(error) {
 118   var event = new Event(hotword.constants.Event.ERROR);
 119   event.data = error;
 120   this.dispatchEvent(event);
 121 };
 122
 123 /**
 124  * Record the result of loading the NaCl plugin to UMA.
 125  * @param {!hotword.constants.UmaNaClPluginLoadResult} error
 126  * @private
 127  */
 128 NaClManager.prototype.logPluginLoadResult_ = function(error) {
 129   hotword.metrics.recordEnum(
 130       hotword.constants.UmaMetrics.NACL_PLUGIN_LOAD_RESULT,
 131       error,
 132       UmaNaClPluginLoadResult_.MAX);
 133 };
 134
 135 /**
 136  * Set a timeout. Only allow one timeout to exist at any given time.
 137  * @param {!function()} func
 138  * @param {number} timeout
 139  * @private
 140  */
 141 NaClManager.prototype.setTimeout_ = function(func, timeout) {
 142   assert(!this.naclTimeoutId_, 'Timeout already exists');
 143   this.naclTimeoutId_ = window.setTimeout(
 144       function() {
 145         this.naclTimeoutId_ = null;
 146         func();
 147       }.bind(this), timeout);
 148 };
 149
 150 /**
 151  * Clears the current timeout.
 152  * @private
 153  */
 154 NaClManager.prototype.clearTimeout_ = function() {
 155   window.clearTimeout(this.naclTimeoutId_);
 156   this.naclTimeoutId_ = null;
 157 };
 158
 159 /**
 160  * Starts a stopped or stopping hotword recognizer (NaCl plugin).
 161  * @param {hotword.constants.RecognizerStartMode} mode The mode to start the
 162  *     recognizer in.
 163  */
 164 NaClManager.prototype.startRecognizer = function(mode) {
 165   this.startMode_ = mode;
 166   if (this.recognizerState_ == ManagerState_.STOPPED) {
 167     this.preambleLog_ = null;
 168     this.recognizerState_ = ManagerState_.STARTING;
 169     if (mode == hotword.constants.RecognizerStartMode.NEW_MODEL) {
 170       hotword.debug('Starting Recognizer in START training mode');
 171       this.sendDataToPlugin_(hotword.constants.NaClPlugin.BEGIN_SPEAKER_MODEL);
 172     } else if (mode == hotword.constants.RecognizerStartMode.ADAPT_MODEL) {
 173       hotword.debug('Starting Recognizer in ADAPT training mode');
 174       this.sendDataToPlugin_(hotword.constants.NaClPlugin.ADAPT_SPEAKER_MODEL);
 175     } else {
 176       hotword.debug('Starting Recognizer in NORMAL mode');
 177       this.sendDataToPlugin_(hotword.constants.NaClPlugin.RESTART);
 178     }
 179     // Normally, there would be a waitForMessage_(READY_FOR_AUDIO) here.
 180     // However, this message is sent the first time audio data is read and in
 181     // some cases (ie. using the hotword stream), this won't happen until a
 182     // potential hotword trigger is seen. Having a waitForMessage_() would time
 183     // out in this case, so just leave it out. This ends up sacrificing a bit of
 184     // error detection in the non-hotword-stream case, but I think we can live
 185     // with that.
 186   } else if (this.recognizerState_ == ManagerState_.STOPPING) {
 187     // Wait until the plugin is stopped before trying to start it.
 188     this.restartOnStop_ = true;
 189   } else {
 190     throw 'Attempting to start NaCl recogniser not in STOPPED or STOPPING ' +
 191         'state';
 192   }
 193 };
 194
 195 /**
 196  * Stops the hotword recognizer.
 197  */
 198 NaClManager.prototype.stopRecognizer = function() {
 199   if (this.recognizerState_ == ManagerState_.STARTING) {
 200     // If the recognizer is stopped before it finishes starting, it causes an
 201     // assertion to be raised in waitForMessage_() since we're waiting for the
 202     // READY_FOR_AUDIO message. Clear the current timeout and expecting message
 203     // since we no longer expect it and may never receive it.
 204     this.clearTimeout_();
 205     this.expectingMessage_ = null;
 206   }
 207   this.sendDataToPlugin_(hotword.constants.NaClPlugin.STOP);
 208   this.recognizerState_ = ManagerState_.STOPPING;
 209   this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
 210                        hotword.constants.NaClPlugin.STOPPED);
 211 };
 212
 213 /**
 214  * Saves the speaker model.
 215  */
 216 NaClManager.prototype.finalizeSpeakerModel = function() {
 217   if (this.recognizerState_ == ManagerState_.UNINITIALIZED ||
 218       this.recognizerState_ == ManagerState_.ERROR ||
 219       this.recognizerState_ == ManagerState_.SHUTDOWN ||
 220       this.recognizerState_ == ManagerState_.LOADING) {
 221     return;
 222   }
 223   this.sendDataToPlugin_(hotword.constants.NaClPlugin.FINISH_SPEAKER_MODEL);
 224 };
 225
 226 /**
 227  * Checks whether the file at the given path exists.
 228  * @param {!string} path Path to a file. Can be any valid URL.
 229  * @return {boolean} True if the patch exists.
 230  * @private
 231  */
 232 NaClManager.prototype.fileExists_ = function(path) {
 233   var xhr = new XMLHttpRequest();
 234   xhr.open('HEAD', path, false);
 235   try {
 236     xhr.send();
 237   } catch (err) {
 238     return false;
 239   }
 240   if (xhr.readyState != xhr.DONE || xhr.status != 200) {
 241     return false;
 242   }
 243   return true;
 244 };
 245
 246 /**
 247  * Creates and returns a list of possible languages to check for hotword
 248  * support.
 249  * @return {!Array<string>} Array of languages.
 250  * @private
 251  */
 252 NaClManager.prototype.getPossibleLanguages_ = function() {
 253   // Create array used to search first for language-country, if not found then
 254   // search for language, if not found then no language (empty string).
 255   // For example, search for 'en-us', then 'en', then ''.
 256   var langs = new Array();
 257   if (hotword.constants.UI_LANGUAGE) {
 258     // Chrome webstore doesn't support uppercase path: crbug.com/353407
 259     var language = hotword.constants.UI_LANGUAGE.toLowerCase();
 260     langs.push(language);  // Example: 'en-us'.
 261     // Remove country to add just the language to array.
 262     var hyphen = language.lastIndexOf('-');
 263     if (hyphen >= 0) {
 264       langs.push(language.substr(0, hyphen));  // Example: 'en'.
 265     }
 266   }
 267   langs.push('');
 268   return langs;
 269 };
 270
 271 /**
 272  * Creates a NaCl plugin object and attaches it to the page.
 273  * @param {!string} src Location of the plugin.
 274  * @return {!HTMLEmbedElement} NaCl plugin DOM object.
 275  * @private
 276  */
 277 NaClManager.prototype.createPlugin_ = function(src) {
 278   var plugin = /** @type {HTMLEmbedElement} */(document.createElement('embed'));
 279   plugin.src = src;
 280   plugin.type = 'application/x-nacl';
 281   document.body.appendChild(plugin);
 282   return plugin;
 283 };
 284
 285 /**
 286  * Initializes the NaCl manager.
 287  * @param {!string} naclArch Either 'arm', 'x86-32' or 'x86-64'.
 288  * @param {!MediaStream} stream A stream containing an audio source track.
 289  * @return {boolean} True if the successful.
 290  */
 291 NaClManager.prototype.initialize = function(naclArch, stream) {
 292   assert(this.recognizerState_ == ManagerState_.UNINITIALIZED,
 293          'Recognizer not in uninitialized state. State: ' +
 294          this.recognizerState_);
 295   var langs = this.getPossibleLanguages_();
 296   var i, j;
 297   // For country-lang variations. For example, when combined with path it will
 298   // attempt to find: '/x86-32_en-gb/', else '/x86-32_en/', else '/x86-32_/'.
 299   for (i = 0; i < langs.length; i++) {
 300     var folder = hotword.constants.SHARED_MODULE_ROOT + '/_platform_specific/' +
 301         naclArch + '_' + langs[i] + '/';
 302     var dataSrc = folder + hotword.constants.File.RECOGNIZER_CONFIG;
 303     var pluginSrc = hotword.constants.SHARED_MODULE_ROOT + '/hotword_' +
 304         langs[i] + '.nmf';
 305     var dataExists = this.fileExists_(dataSrc) && this.fileExists_(pluginSrc);
 306     if (!dataExists) {
 307       continue;
 308     }
 309
 310     var plugin = this.createPlugin_(pluginSrc);
 311     this.plugin_ = plugin;
 312     if (!this.plugin_ || !this.plugin_.postMessage) {
 313       document.body.removeChild(this.plugin_);
 314       this.recognizerState_ = ManagerState_.ERROR;
 315       return false;
 316     }
 317     this.modelUrl_ = chrome.extension.getURL(dataSrc);
 318     this.stream_ = stream;
 319     this.recognizerState_ = ManagerState_.LOADING;
 320
 321     plugin.addEventListener('message',
 322                             this.handlePluginMessage_.bind(this),
 323                             false);
 324
 325     plugin.addEventListener('crash',
 326                             function() {
 327                               this.handleError_(Error_.NACL_CRASH);
 328                               this.logPluginLoadResult_(
 329                                   UmaNaClPluginLoadResult_.CRASH);
 330                             }.bind(this),
 331                             false);
 332     return true;
 333   }
 334   this.recognizerState_ = ManagerState_.ERROR;
 335   this.logPluginLoadResult_(UmaNaClPluginLoadResult_.NO_MODULE_FOUND);
 336   return false;
 337 };
 338
 339 /**
 340  * Shuts down the NaCl plugin and frees all resources.
 341  */
 342 NaClManager.prototype.shutdown = function() {
 343   if (this.plugin_ != null) {
 344     document.body.removeChild(this.plugin_);
 345     this.plugin_ = null;
 346   }
 347   this.clearTimeout_();
 348   this.recognizerState_ = ManagerState_.SHUTDOWN;
 349   if (this.stream_)
 350     this.stream_.stop();
 351   this.stream_ = null;
 352 };
 353
 354 /**
 355  * Sends data to the NaCl plugin.
 356  * @param {!string|!MediaStreamTrack} data Command to be sent to NaCl plugin.
 357  * @private
 358  */
 359 NaClManager.prototype.sendDataToPlugin_ = function(data) {
 360   assert(this.recognizerState_ != ManagerState_.UNINITIALIZED,
 361          'Recognizer in uninitialized state');
 362   this.plugin_.postMessage(data);
 363 };
 364
 365 /**
 366  * Waits, with a timeout, for a message to be received from the plugin. If the
 367  * message is not seen within the timeout, dispatch an 'error' event and go into
 368  * the ERROR state.
 369  * @param {number} timeout Timeout, in milliseconds, to wait for the message.
 370  * @param {!string} message Message to wait for.
 371  * @private
 372  */
 373 NaClManager.prototype.waitForMessage_ = function(timeout, message) {
 374   assert(this.expectingMessage_ == null, 'Cannot wait for message: ' +
 375       message + ', already waiting for message ' + this.expectingMessage_);
 376   this.setTimeout_(
 377       function() {
 378         this.recognizerState_ = ManagerState_.ERROR;
 379         this.handleError_(Error_.TIMEOUT);
 380         switch (this.expectingMessage_) {
 381           case hotword.constants.NaClPlugin.REQUEST_MODEL:
 382             var metricValue = UmaNaClMessageTimeout_.REQUEST_MODEL;
 383             break;
 384           case hotword.constants.NaClPlugin.MODEL_LOADED:
 385             var metricValue = UmaNaClMessageTimeout_.MODEL_LOADED;
 386             break;
 387           case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
 388             var metricValue = UmaNaClMessageTimeout_.READY_FOR_AUDIO;
 389             break;
 390           case hotword.constants.NaClPlugin.STOPPED:
 391             var metricValue = UmaNaClMessageTimeout_.STOPPED;
 392             break;
 393           case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
 394             var metricValue = UmaNaClMessageTimeout_.HOTWORD_DETECTED;
 395             break;
 396           case hotword.constants.NaClPlugin.MS_CONFIGURED:
 397             var metricValue = UmaNaClMessageTimeout_.MS_CONFIGURED;
 398             break;
 399         }
 400         hotword.metrics.recordEnum(
 401             hotword.constants.UmaMetrics.NACL_MESSAGE_TIMEOUT,
 402             metricValue,
 403             UmaNaClMessageTimeout_.MAX);
 404       }.bind(this), timeout);
 405   this.expectingMessage_ = message;
 406 };
 407
 408 /**
 409  * Called when a message is received from the plugin. If we're waiting for that
 410  * message, cancel the pending timeout.
 411  * @param {string} message Message received.
 412  * @private
 413  */
 414 NaClManager.prototype.receivedMessage_ = function(message) {
 415   if (message == this.expectingMessage_) {
 416     this.clearTimeout_();
 417     this.expectingMessage_ = null;
 418   }
 419 };
 420
 421 /**
 422  * Handle a REQUEST_MODEL message from the plugin.
 423  * The plugin sends this message immediately after starting.
 424  * @private
 425  */
 426 NaClManager.prototype.handleRequestModel_ = function() {
 427   if (this.recognizerState_ != ManagerState_.LOADING) {
 428     return;
 429   }
 430   this.logPluginLoadResult_(UmaNaClPluginLoadResult_.SUCCESS);
 431   this.sendDataToPlugin_(
 432       hotword.constants.NaClPlugin.MODEL_PREFIX + this.modelUrl_);
 433   this.waitForMessage_(hotword.constants.TimeoutMs.LONG,
 434                        hotword.constants.NaClPlugin.MODEL_LOADED);
 435
 436   // Configure logging in the plugin. This can be configured any time before
 437   // starting the recognizer, and now is as good a time as any.
 438   if (this.loggingEnabled_) {
 439     this.sendDataToPlugin_(
 440         hotword.constants.NaClPlugin.LOG + ':' +
 441         hotword.constants.AUDIO_LOG_SECONDS);
 442   }
 443
 444   // If the audio stream is from a hotword stream, tell the plugin.
 445   if (this.hotwordStream_) {
 446     this.sendDataToPlugin_(
 447         hotword.constants.NaClPlugin.DSP + ':' +
 448         hotword.constants.HOTWORD_STREAM_TIMEOUT_SECONDS);
 449   }
 450 };
 451
 452 /**
 453  * Handle a MODEL_LOADED message from the plugin.
 454  * The plugin sends this message after successfully loading the language model.
 455  * @private
 456  */
 457 NaClManager.prototype.handleModelLoaded_ = function() {
 458   if (this.recognizerState_ != ManagerState_.LOADING) {
 459     return;
 460   }
 461   this.sendDataToPlugin_(this.stream_.getAudioTracks()[0]);
 462   this.waitForMessage_(hotword.constants.TimeoutMs.LONG,
 463                        hotword.constants.NaClPlugin.MS_CONFIGURED);
 464 };
 465
 466 /**
 467  * Handle a MS_CONFIGURED message from the plugin.
 468  * The plugin sends this message after successfully configuring the audio input
 469  * stream.
 470  * @private
 471  */
 472 NaClManager.prototype.handleMsConfigured_ = function() {
 473   if (this.recognizerState_ != ManagerState_.LOADING) {
 474     return;
 475   }
 476   this.recognizerState_ = ManagerState_.STOPPED;
 477   this.dispatchEvent(new Event(hotword.constants.Event.READY));
 478 };
 479
 480 /**
 481  * Handle a READY_FOR_AUDIO message from the plugin.
 482  * The plugin sends this message after the recognizer is started and
 483  * successfully receives and processes audio data.
 484  * @private
 485  */
 486 NaClManager.prototype.handleReadyForAudio_ = function() {
 487   if (this.recognizerState_ != ManagerState_.STARTING) {
 488     return;
 489   }
 490   this.recognizerState_ = ManagerState_.RUNNING;
 491 };
 492
 493 /**
 494  * Handle a HOTWORD_DETECTED message from the plugin.
 495  * The plugin sends this message after detecting the hotword.
 496  * @private
 497  */
 498 NaClManager.prototype.handleHotwordDetected_ = function() {
 499   if (this.recognizerState_ != ManagerState_.RUNNING) {
 500     return;
 501   }
 502   // We'll receive a STOPPED message very soon.
 503   this.recognizerState_ = ManagerState_.STOPPING;
 504   this.waitForMessage_(hotword.constants.TimeoutMs.NORMAL,
 505                        hotword.constants.NaClPlugin.STOPPED);
 506   var event = new Event(hotword.constants.Event.TRIGGER);
 507   event.log = this.preambleLog_;
 508   this.dispatchEvent(event);
 509 };
 510
 511 /**
 512  * Handle a STOPPED message from the plugin.
 513  * This plugin sends this message after stopping the recognizer. This can happen
 514  * either in response to a stop request, or after the hotword is detected.
 515  * @private
 516  */
 517 NaClManager.prototype.handleStopped_ = function() {
 518   this.recognizerState_ = ManagerState_.STOPPED;
 519   if (this.restartOnStop_) {
 520     this.restartOnStop_ = false;
 521     this.startRecognizer(this.startMode_);
 522   }
 523 };
 524
 525 /**
 526  * Handle a TIMEOUT message from the plugin.
 527  * The plugin sends this message when it thinks the stream is from a DSP and
 528  * a hotword wasn't detected within a timeout period after arrival of the first
 529  * audio samples.
 530  * @private
 531  */
 532 NaClManager.prototype.handleTimeout_ = function() {
 533   if (this.recognizerState_ != ManagerState_.RUNNING) {
 534     return;
 535   }
 536   this.recognizerState_ = ManagerState_.STOPPED;
 537   this.dispatchEvent(new Event(hotword.constants.Event.TIMEOUT));
 538 };
 539
 540 /**
 541  * Handle a SPEAKER_MODEL_SAVED message from the plugin.
 542  * The plugin sends this message after writing the model to a file.
 543  * @private
 544  */
 545 NaClManager.prototype.handleSpeakerModelSaved_ = function() {
 546   this.dispatchEvent(new Event(hotword.constants.Event.SPEAKER_MODEL_SAVED));
 547 };
 548
 549 /**
 550  * Handles a message from the NaCl plugin.
 551  * @param {!Event} msg Message from NaCl plugin.
 552  * @private
 553  */
 554 NaClManager.prototype.handlePluginMessage_ = function(msg) {
 555   if (msg['data']) {
 556     if (typeof(msg['data']) == 'object') {
 557       // Save the preamble for delivery to the trigger handler when the trigger
 558       // message arrives.
 559       this.preambleLog_ = msg['data'];
 560       return;
 561     }
 562     this.receivedMessage_(msg['data']);
 563     switch (msg['data']) {
 564       case hotword.constants.NaClPlugin.REQUEST_MODEL:
 565         this.handleRequestModel_();
 566         break;
 567       case hotword.constants.NaClPlugin.MODEL_LOADED:
 568         this.handleModelLoaded_();
 569         break;
 570       case hotword.constants.NaClPlugin.MS_CONFIGURED:
 571         this.handleMsConfigured_();
 572         break;
 573       case hotword.constants.NaClPlugin.READY_FOR_AUDIO:
 574         this.handleReadyForAudio_();
 575         break;
 576       case hotword.constants.NaClPlugin.HOTWORD_DETECTED:
 577         this.handleHotwordDetected_();
 578         break;
 579       case hotword.constants.NaClPlugin.STOPPED:
 580         this.handleStopped_();
 581         break;
 582       case hotword.constants.NaClPlugin.TIMEOUT:
 583         this.handleTimeout_();
 584         break;
 585       case hotword.constants.NaClPlugin.SPEAKER_MODEL_SAVED:
 586         this.handleSpeakerModelSaved_();
 587         break;
 588     }
 589   }
 590 };
 591
 592 return {
 593   NaClManager: NaClManager
 594 };
 595
 596 });