chrome/browser/resources/hotword/state_manager.js

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 cr.define('hotword', function() {
   6   'use strict';
   7
   8   /**
   9    * Trivial container class for session information.
  10    * @param {!hotword.constants.SessionSource} source Source of the hotword
  11    *     session.
  12    * @param {!function()} triggerCb Callback invoked when the hotword has
  13    *     triggered.
  14    * @param {!function()} startedCb Callback invoked when the session has
  15    *     been started successfully.
  16    * @param {function()=} opt_modelSavedCb Callback invoked when the speaker
  17    *     model has been saved successfully.
  18    * @constructor
  19    * @struct
  20    * @private
  21    */
  22   function Session_(source, triggerCb, startedCb, opt_modelSavedCb) {
  23     /**
  24      * Source of the hotword session request.
  25      * @private {!hotword.constants.SessionSource}
  26      */
  27     this.source_ = source;
  28
  29      /**
  30       * Callback invoked when the hotword has triggered.
  31       * @private {!function()}
  32       */
  33     this.triggerCb_ = triggerCb;
  34
  35     /**
  36      * Callback invoked when the session has been started successfully.
  37      * @private {?function()}
  38      */
  39     this.startedCb_ = startedCb;
  40
  41     /**
  42      * Callback invoked when the session has been started successfully.
  43      * @private {?function()}
  44      */
  45     this.speakerModelSavedCb_ = opt_modelSavedCb;
  46   }
  47
  48   /**
  49    * Class to manage hotwording state. Starts/stops the hotword detector based
  50    * on user settings, session requests, and any other factors that play into
  51    * whether or not hotwording should be running.
  52    * @constructor
  53    */
  54   function StateManager() {
  55     /**
  56      * Current state.
  57      * @private {hotword.StateManager.State_}
  58      */
  59     this.state_ = State_.STOPPED;
  60
  61     /**
  62      * Current hotwording status.
  63      * @private {?chrome.hotwordPrivate.StatusDetails}
  64      */
  65     this.hotwordStatus_ = null;
  66
  67     /**
  68      * NaCl plugin manager.
  69      * @private {?hotword.NaClManager}
  70      */
  71     this.pluginManager_ = null;
  72
  73     /**
  74      * Currently active hotwording sessions.
  75      * @private {!Array<Session_>}
  76      */
  77     this.sessions_ = [];
  78
  79     /**
  80      * The mode to start the recognizer in.
  81      * @private {!hotword.constants.RecognizerStartMode}
  82      */
  83     this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;
  84
  85     /**
  86      * Event that fires when the hotwording status has changed.
  87      * @type {!ChromeEvent}
  88      */
  89     this.onStatusChanged = new chrome.Event();
  90
  91     /**
  92      * Hotword trigger audio notification... a.k.a The Chime (tm).
  93      * @private {!HTMLAudioElement}
  94      */
  95     this.chime_ =
  96         /** @type {!HTMLAudioElement} */(document.createElement('audio'));
  97
  98     /**
  99      * Chrome event listeners. Saved so that they can be de-registered when
 100      * hotwording is disabled.
 101      * @private
 102      */
 103     this.idleStateChangedListener_ = this.handleIdleStateChanged_.bind(this);
 104
 105     /**
 106      * Whether this user is locked.
 107      * @private {boolean}
 108      */
 109     this.isLocked_ = false;
 110
 111     /**
 112      * Current state of audio logging.
 113      * This is tracked separately from hotwordStatus_ because we need to restart
 114      * the hotword detector when this value changes.
 115      * @private {boolean}
 116      */
 117     this.loggingEnabled_ = false;
 118
 119     /**
 120      * Current state of training.
 121      * This is tracked separately from |hotwordStatus_| because we need to
 122      * restart the hotword detector when this value changes.
 123      * @private {!boolean}
 124      */
 125     this.trainingEnabled_ = false;
 126
 127     /**
 128      * Helper class to keep this extension alive while the hotword detector is
 129      * running in always-on mode.
 130      * @private {!hotword.KeepAlive}
 131      */
 132     this.keepAlive_ = new hotword.KeepAlive();
 133
 134     // Get the initial status.
 135     chrome.hotwordPrivate.getStatus(this.handleStatus_.bind(this));
 136
 137     // Setup the chime and insert into the page.
 138     // Set preload=none to prevent an audio output stream from being created
 139     // when the extension loads.
 140     this.chime_.preload = 'none';
 141     this.chime_.src = chrome.extension.getURL(
 142         hotword.constants.SHARED_MODULE_ROOT + '/audio/chime.wav');
 143     document.body.appendChild(this.chime_);
 144   }
 145
 146   /**
 147    * @enum {number}
 148    * @private
 149    */
 150   StateManager.State_ = {
 151     STOPPED: 0,
 152     STARTING: 1,
 153     RUNNING: 2,
 154     ERROR: 3,
 155   };
 156   var State_ = StateManager.State_;
 157
 158   var UmaMediaStreamOpenResults_ = {
 159     // These first error are defined by the MediaStream spec:
 160     // http://w3c.github.io/mediacapture-main/getusermedia.html#idl-def-MediaStreamError
 161     'NotSupportedError':
 162         hotword.constants.UmaMediaStreamOpenResult.NOT_SUPPORTED,
 163     'PermissionDeniedError':
 164         hotword.constants.UmaMediaStreamOpenResult.PERMISSION_DENIED,
 165     'ConstraintNotSatisfiedError':
 166         hotword.constants.UmaMediaStreamOpenResult.CONSTRAINT_NOT_SATISFIED,
 167     'OverconstrainedError':
 168         hotword.constants.UmaMediaStreamOpenResult.OVERCONSTRAINED,
 169     'NotFoundError': hotword.constants.UmaMediaStreamOpenResult.NOT_FOUND,
 170     'AbortError': hotword.constants.UmaMediaStreamOpenResult.ABORT,
 171     'SourceUnavailableError':
 172         hotword.constants.UmaMediaStreamOpenResult.SOURCE_UNAVAILABLE,
 173     // The next few errors are chrome-specific. See:
 174     // content/renderer/media/user_media_client_impl.cc
 175     // (UserMediaClientImpl::GetUserMediaRequestFailed)
 176     'PermissionDismissedError':
 177         hotword.constants.UmaMediaStreamOpenResult.PERMISSION_DISMISSED,
 178     'InvalidStateError':
 179         hotword.constants.UmaMediaStreamOpenResult.INVALID_STATE,
 180     'DevicesNotFoundError':
 181         hotword.constants.UmaMediaStreamOpenResult.DEVICES_NOT_FOUND,
 182     'InvalidSecurityOriginError':
 183         hotword.constants.UmaMediaStreamOpenResult.INVALID_SECURITY_ORIGIN
 184   };
 185
 186   var UmaTriggerSources_ = {
 187     'launcher': hotword.constants.UmaTriggerSource.LAUNCHER,
 188     'ntp': hotword.constants.UmaTriggerSource.NTP_GOOGLE_COM,
 189     'always': hotword.constants.UmaTriggerSource.ALWAYS_ON,
 190     'training': hotword.constants.UmaTriggerSource.TRAINING
 191   };
 192
 193   StateManager.prototype = {
 194     /**
 195      * Request status details update. Intended to be called from the
 196      * hotwordPrivate.onEnabledChanged() event.
 197      */
 198     updateStatus: function() {
 199       chrome.hotwordPrivate.getStatus(this.handleStatus_.bind(this));
 200     },
 201
 202     /**
 203      * @return {boolean} True if google.com/NTP/launcher hotwording is enabled.
 204      */
 205     isSometimesOnEnabled: function() {
 206       assert(this.hotwordStatus_,
 207              'No hotwording status (isSometimesOnEnabled)');
 208       // Although the two settings are supposed to be mutually exclusive, it's
 209       // possible for both to be set. In that case, always-on takes precedence.
 210       return this.hotwordStatus_.enabled &&
 211           !this.hotwordStatus_.alwaysOnEnabled;
 212     },
 213
 214     /**
 215      * @return {boolean} True if always-on hotwording is enabled.
 216      */
 217     isAlwaysOnEnabled: function() {
 218       assert(this.hotwordStatus_, 'No hotword status (isAlwaysOnEnabled)');
 219       return this.hotwordStatus_.alwaysOnEnabled &&
 220           !this.hotwordStatus_.trainingEnabled;
 221     },
 222
 223     /**
 224      * @return {boolean} True if training is enabled.
 225      */
 226     isTrainingEnabled: function() {
 227       assert(this.hotwordStatus_, 'No hotword status (isTrainingEnabled)');
 228       return this.hotwordStatus_.trainingEnabled;
 229     },
 230
 231     /**
 232      * Callback for hotwordPrivate.getStatus() function.
 233      * @param {chrome.hotwordPrivate.StatusDetails} status Current hotword
 234      *     status.
 235      * @private
 236      */
 237     handleStatus_: function(status) {
 238       hotword.debug('New hotword status', status);
 239       this.hotwordStatus_ = status;
 240       this.updateStateFromStatus_();
 241
 242       this.onStatusChanged.dispatch();
 243     },
 244
 245     /**
 246      * Updates state based on the current status.
 247      * @private
 248      */
 249     updateStateFromStatus_: function() {
 250       if (!this.hotwordStatus_)
 251         return;
 252
 253       if (this.hotwordStatus_.enabled ||
 254           this.hotwordStatus_.alwaysOnEnabled ||
 255           this.hotwordStatus_.trainingEnabled) {
 256         // Detect changes to audio logging and kill the detector if that setting
 257         // has changed.
 258         if (this.hotwordStatus_.audioLoggingEnabled != this.loggingEnabled_)
 259           this.shutdownDetector_();
 260         this.loggingEnabled_ = this.hotwordStatus_.audioLoggingEnabled;
 261
 262         // If the training state has changed, we need to first shut down the
 263         // detector so that we can restart in a different mode.
 264         if (this.hotwordStatus_.trainingEnabled != this.trainingEnabled_)
 265           this.shutdownDetector_();
 266         this.trainingEnabled_ = this.hotwordStatus_.trainingEnabled;
 267
 268         // Start the detector if there's a session and the user is unlocked, and
 269         // stops it otherwise.
 270         if (this.sessions_.length && !this.isLocked_ &&
 271             this.hotwordStatus_.userIsActive) {
 272           this.startDetector_();
 273         } else {
 274           this.shutdownDetector_();
 275         }
 276
 277         if (!chrome.idle.onStateChanged.hasListener(
 278                 this.idleStateChangedListener_)) {
 279           chrome.idle.onStateChanged.addListener(
 280               this.idleStateChangedListener_);
 281         }
 282       } else {
 283         // Not enabled. Shut down if running.
 284         this.shutdownDetector_();
 285
 286         chrome.idle.onStateChanged.removeListener(
 287             this.idleStateChangedListener_);
 288       }
 289     },
 290
 291     /**
 292      * Starts the hotword detector.
 293      * @private
 294      */
 295     startDetector_: function() {
 296       // Last attempt to start detector resulted in an error.
 297       if (this.state_ == State_.ERROR) {
 298         // TODO(amistry): Do some error rate tracking here and disable the
 299         // extension if we error too often.
 300       }
 301
 302       if (!this.pluginManager_) {
 303         this.state_ = State_.STARTING;
 304         var isHotwordStream = this.isAlwaysOnEnabled() &&
 305             this.hotwordStatus_.hotwordHardwareAvailable;
 306         this.pluginManager_ = new hotword.NaClManager(this.loggingEnabled_,
 307                                                       isHotwordStream);
 308         this.pluginManager_.addEventListener(hotword.constants.Event.READY,
 309                                              this.onReady_.bind(this));
 310         this.pluginManager_.addEventListener(hotword.constants.Event.ERROR,
 311                                              this.onError_.bind(this));
 312         this.pluginManager_.addEventListener(hotword.constants.Event.TRIGGER,
 313                                              this.onTrigger_.bind(this));
 314         this.pluginManager_.addEventListener(hotword.constants.Event.TIMEOUT,
 315                                              this.onTimeout_.bind(this));
 316         this.pluginManager_.addEventListener(
 317             hotword.constants.Event.SPEAKER_MODEL_SAVED,
 318             this.onSpeakerModelSaved_.bind(this));
 319         chrome.runtime.getPlatformInfo(function(platform) {
 320           var naclArch = platform.nacl_arch;
 321
 322           // googDucking set to false so that audio output level from other tabs
 323           // is not affected when hotword is enabled. https://crbug.com/357773
 324           // content/common/media/media_stream_options.cc
 325           // When always-on is enabled, request the hotword stream.
 326           // Optional because we allow power users to bypass the hardware
 327           // detection via a flag, and hence the hotword stream may not be
 328           // available.
 329           var constraints = /** @type {googMediaStreamConstraints} */
 330               ({audio: {optional: [
 331                 { googDucking: false },
 332                 { googHotword: this.isAlwaysOnEnabled() }
 333               ]}});
 334           navigator.webkitGetUserMedia(
 335               /** @type {MediaStreamConstraints} */ (constraints),
 336               function(stream) {
 337                 hotword.metrics.recordEnum(
 338                     hotword.constants.UmaMetrics.MEDIA_STREAM_RESULT,
 339                     hotword.constants.UmaMediaStreamOpenResult.SUCCESS,
 340                     hotword.constants.UmaMediaStreamOpenResult.MAX);
 341                 // The detector could have been shut down before the stream
 342                 // finishes opening.
 343                 if (this.pluginManager_ == null) {
 344                   stream.getAudioTracks()[0].stop();
 345                   return;
 346                 }
 347
 348                 if (this.isAlwaysOnEnabled())
 349                   this.keepAlive_.start();
 350                 if (!this.pluginManager_.initialize(naclArch, stream)) {
 351                   this.state_ = State_.ERROR;
 352                   this.shutdownPluginManager_();
 353                 }
 354               }.bind(this),
 355               function(error) {
 356                 if (error.name in UmaMediaStreamOpenResults_) {
 357                   var metricValue = UmaMediaStreamOpenResults_[error.name];
 358                 } else {
 359                   var metricValue =
 360                       hotword.constants.UmaMediaStreamOpenResult.UNKNOWN;
 361                 }
 362                 hotword.metrics.recordEnum(
 363                     hotword.constants.UmaMetrics.MEDIA_STREAM_RESULT,
 364                     metricValue,
 365                     hotword.constants.UmaMediaStreamOpenResult.MAX);
 366                 this.state_ = State_.ERROR;
 367                 this.pluginManager_ = null;
 368               }.bind(this));
 369         }.bind(this));
 370       } else if (this.state_ != State_.STARTING) {
 371         // Don't try to start a starting detector.
 372         this.startRecognizer_();
 373       }
 374     },
 375
 376     /**
 377      * Start the recognizer plugin. Assumes the plugin has been loaded and is
 378      * ready to start.
 379      * @private
 380      */
 381     startRecognizer_: function() {
 382       assert(this.pluginManager_, 'No NaCl plugin loaded');
 383       if (this.state_ != State_.RUNNING) {
 384         this.state_ = State_.RUNNING;
 385         if (this.isAlwaysOnEnabled())
 386           this.keepAlive_.start();
 387         this.pluginManager_.startRecognizer(this.startMode_);
 388       }
 389       for (var i = 0; i < this.sessions_.length; i++) {
 390         var session = this.sessions_[i];
 391         if (session.startedCb_) {
 392           session.startedCb_();
 393           session.startedCb_ = null;
 394         }
 395       }
 396     },
 397
 398     /**
 399      * Stops the hotword detector, if it's running.
 400      * @private
 401      */
 402     stopDetector_: function() {
 403       this.keepAlive_.stop();
 404       if (this.pluginManager_ && this.state_ == State_.RUNNING) {
 405         this.state_ = State_.STOPPED;
 406         this.pluginManager_.stopRecognizer();
 407       }
 408     },
 409
 410     /**
 411      * Shuts down and removes the plugin manager, if it exists.
 412      * @private
 413      */
 414     shutdownPluginManager_: function() {
 415       this.keepAlive_.stop();
 416       if (this.pluginManager_) {
 417         this.pluginManager_.shutdown();
 418         this.pluginManager_ = null;
 419       }
 420     },
 421
 422     /**
 423      * Shuts down the hotword detector.
 424      * @private
 425      */
 426     shutdownDetector_: function() {
 427       this.state_ = State_.STOPPED;
 428       this.shutdownPluginManager_();
 429     },
 430
 431     /**
 432      * Finalizes the speaker model. Assumes the plugin has been loaded and
 433      * started.
 434      */
 435     finalizeSpeakerModel: function() {
 436       assert(this.pluginManager_,
 437              'Cannot finalize speaker model: No NaCl plugin loaded');
 438       if (this.state_ != State_.RUNNING) {
 439         hotword.debug('Cannot finalize speaker model: NaCl plugin not started');
 440         return;
 441       }
 442       this.pluginManager_.finalizeSpeakerModel();
 443     },
 444
 445     /**
 446      * Handle the hotword plugin being ready to start.
 447      * @private
 448      */
 449     onReady_: function() {
 450       if (this.state_ != State_.STARTING) {
 451         // At this point, we should not be in the RUNNING state. Doing so would
 452         // imply the hotword detector was started without being ready.
 453         assert(this.state_ != State_.RUNNING, 'Unexpected RUNNING state');
 454         this.shutdownPluginManager_();
 455         return;
 456       }
 457       this.startRecognizer_();
 458     },
 459
 460     /**
 461      * Handle an error from the hotword plugin.
 462      * @private
 463      */
 464     onError_: function() {
 465       this.state_ = State_.ERROR;
 466       this.shutdownPluginManager_();
 467     },
 468
 469     /**
 470      * Handle hotword triggering.
 471      * @param {!Event} event Event containing audio log data.
 472      * @private
 473      */
 474     onTrigger_: function(event) {
 475       this.keepAlive_.stop();
 476       hotword.debug('Hotword triggered!');
 477       chrome.metricsPrivate.recordUserAction(
 478           hotword.constants.UmaMetrics.TRIGGER);
 479       assert(this.pluginManager_, 'No NaCl plugin loaded on trigger');
 480       // Detector implicitly stops when the hotword is detected.
 481       this.state_ = State_.STOPPED;
 482
 483       // Play the chime.
 484       this.chime_.play();
 485
 486       // Implicitly clear the top session. A session needs to be started in
 487       // order to restart the detector.
 488       if (this.sessions_.length) {
 489         var session = this.sessions_.pop();
 490         session.triggerCb_(event.log);
 491
 492         hotword.metrics.recordEnum(
 493             hotword.constants.UmaMetrics.TRIGGER_SOURCE,
 494             UmaTriggerSources_[session.source_],
 495             hotword.constants.UmaTriggerSource.MAX);
 496       }
 497
 498       // If we're in always-on mode, shut down the hotword detector. The hotword
 499       // stream requires that we close and re-open it after a trigger, and the
 500       // only way to accomplish this is to shut everything down.
 501       if (this.isAlwaysOnEnabled())
 502         this.shutdownDetector_();
 503     },
 504
 505     /**
 506      * Handle hotword timeout.
 507      * @private
 508      */
 509     onTimeout_: function() {
 510       hotword.debug('Hotword timeout!');
 511
 512       // We get this event when the hotword detector thinks there's a false
 513       // trigger. In this case, we need to shut down and restart the detector to
 514       // re-arm the DSP.
 515       this.shutdownDetector_();
 516       this.updateStateFromStatus_();
 517     },
 518
 519     /**
 520      * Handle speaker model saved.
 521      * @private
 522      */
 523     onSpeakerModelSaved_: function() {
 524       hotword.debug('Speaker model saved!');
 525
 526       if (this.sessions_.length) {
 527         // Only call the callback of the the top session.
 528         var session = this.sessions_[this.sessions_.length - 1];
 529         if (session.speakerModelSavedCb_)
 530           session.speakerModelSavedCb_();
 531       }
 532     },
 533
 534     /**
 535      * Remove a hotwording session from the given source.
 536      * @param {!hotword.constants.SessionSource} source Source of the hotword
 537      *     session request.
 538      * @private
 539      */
 540     removeSession_: function(source) {
 541       for (var i = 0; i < this.sessions_.length; i++) {
 542         if (this.sessions_[i].source_ == source) {
 543           this.sessions_.splice(i, 1);
 544           break;
 545         }
 546       }
 547     },
 548
 549     /**
 550      * Start a hotwording session.
 551      * @param {!hotword.constants.SessionSource} source Source of the hotword
 552      *     session request.
 553      * @param {!function()} startedCb Callback invoked when the session has
 554      *     been started successfully.
 555      * @param {!function()} triggerCb Callback invoked when the hotword has
 556      * @param {function()=} modelSavedCb Callback invoked when the speaker model
 557      *     has been saved.
 558      * @param {hotword.constants.RecognizerStartMode=} opt_mode The mode to
 559      *     start the recognizer in.
 560      */
 561     startSession: function(source, startedCb, triggerCb,
 562                            opt_modelSavedCb, opt_mode) {
 563       if (this.isTrainingEnabled() && opt_mode) {
 564         this.startMode_ = opt_mode;
 565       } else {
 566         this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;
 567       }
 568       hotword.debug('Starting session for source: ' + source);
 569       this.removeSession_(source);
 570       this.sessions_.push(new Session_(source, triggerCb, startedCb,
 571                                        opt_modelSavedCb));
 572       this.updateStateFromStatus_();
 573     },
 574
 575     /**
 576      * Stops a hotwording session.
 577      * @param {!hotword.constants.SessionSource} source Source of the hotword
 578      *     session request.
 579      */
 580     stopSession: function(source) {
 581       hotword.debug('Stopping session for source: ' + source);
 582       this.removeSession_(source);
 583       // If this is a training session then switch the start mode back to
 584       // normal.
 585       if (source == hotword.constants.SessionSource.TRAINING)
 586         this.startMode_ = hotword.constants.RecognizerStartMode.NORMAL;
 587       this.updateStateFromStatus_();
 588     },
 589
 590     /**
 591      * Handles a chrome.idle.onStateChanged event.
 592      * @param {!string} state State, one of "active", "idle", or "locked".
 593      * @private
 594      */
 595     handleIdleStateChanged_: function(state) {
 596       hotword.debug('Idle state changed: ' + state);
 597       var oldLocked = this.isLocked_;
 598       if (state == 'locked')
 599         this.isLocked_ = true;
 600       else
 601         this.isLocked_ = false;
 602
 603       if (oldLocked != this.isLocked_)
 604         this.updateStateFromStatus_();
 605     }
 606   };
 607
 608   return {
 609     StateManager: StateManager
 610   };
 611 });