content/browser/speech/speech_recognizer_impl.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/speech/speech_recognizer_impl.h"
   6
   7 #include "base/basictypes.h"
   8 #include "base/bind.h"
   9 #include "base/time/time.h"
  10 #include "content/browser/browser_main_loop.h"
  11 #include "content/browser/media/media_internals.h"
  12 #include "content/browser/speech/audio_buffer.h"
  13 #include "content/browser/speech/google_one_shot_remote_engine.h"
  14 #include "content/public/browser/speech_recognition_event_listener.h"
  15 #include "media/base/audio_converter.h"
  16 #include "net/url_request/url_request_context_getter.h"
  17
  18 #if defined(OS_WIN)
  19 #include "media/audio/win/core_audio_util_win.h"
  20 #endif
  21
  22 using media::AudioBus;
  23 using media::AudioConverter;
  24 using media::AudioInputController;
  25 using media::AudioManager;
  26 using media::AudioParameters;
  27 using media::ChannelLayout;
  28
  29 namespace content {
  30
  31 // Private class which encapsulates the audio converter and the
  32 // AudioConverter::InputCallback. It handles resampling, buffering and
  33 // channel mixing between input and output parameters.
  34 class SpeechRecognizerImpl::OnDataConverter
  35     : public media::AudioConverter::InputCallback {
  36  public:
  37   OnDataConverter(const AudioParameters& input_params,
  38                   const AudioParameters& output_params);
  39   ~OnDataConverter() override;
  40
  41   // Converts input audio |data| bus into an AudioChunk where the input format
  42   // is given by |input_parameters_| and the output format by
  43   // |output_parameters_|.
  44   scoped_refptr<AudioChunk> Convert(const AudioBus* data);
  45
  46  private:
  47   // media::AudioConverter::InputCallback implementation.
  48   double ProvideInput(AudioBus* dest, base::TimeDelta buffer_delay) override;
  49
  50   // Handles resampling, buffering, and channel mixing between input and output
  51   // parameters.
  52   AudioConverter audio_converter_;
  53
  54   scoped_ptr<AudioBus> input_bus_;
  55   scoped_ptr<AudioBus> output_bus_;
  56   const AudioParameters input_parameters_;
  57   const AudioParameters output_parameters_;
  58   bool waiting_for_input_;
  59   scoped_ptr<uint8[]> converted_data_;
  60
  61   DISALLOW_COPY_AND_ASSIGN(OnDataConverter);
  62 };
  63
  64 namespace {
  65
  66 // The following constants are related to the volume level indicator shown in
  67 // the UI for recorded audio.
  68 // Multiplier used when new volume is greater than previous level.
  69 const float kUpSmoothingFactor = 1.0f;
  70 // Multiplier used when new volume is lesser than previous level.
  71 const float kDownSmoothingFactor = 0.7f;
  72 // RMS dB value of a maximum (unclipped) sine wave for int16 samples.
  73 const float kAudioMeterMaxDb = 90.31f;
  74 // This value corresponds to RMS dB for int16 with 6 most-significant-bits = 0.
  75 // Values lower than this will display as empty level-meter.
  76 const float kAudioMeterMinDb = 30.0f;
  77 const float kAudioMeterDbRange = kAudioMeterMaxDb - kAudioMeterMinDb;
  78
  79 // Maximum level to draw to display unclipped meter. (1.0f displays clipping.)
  80 const float kAudioMeterRangeMaxUnclipped = 47.0f / 48.0f;
  81
  82 // Returns true if more than 5% of the samples are at min or max value.
  83 bool DetectClipping(const AudioChunk& chunk) {
  84   const int num_samples = chunk.NumSamples();
  85   const int16* samples = chunk.SamplesData16();
  86   const int kThreshold = num_samples / 20;
  87   int clipping_samples = 0;
  88
  89   for (int i = 0; i < num_samples; ++i) {
  90     if (samples[i] <= -32767 || samples[i] >= 32767) {
  91       if (++clipping_samples > kThreshold)
  92         return true;
  93     }
  94   }
  95   return false;
  96 }
  97
  98 void KeepAudioControllerRefcountedForDtor(scoped_refptr<AudioInputController>) {
  99 }
 100
 101 }  // namespace
 102
 103 const int SpeechRecognizerImpl::kAudioSampleRate = 16000;
 104 const ChannelLayout SpeechRecognizerImpl::kChannelLayout =
 105     media::CHANNEL_LAYOUT_MONO;
 106 const int SpeechRecognizerImpl::kNumBitsPerAudioSample = 16;
 107 const int SpeechRecognizerImpl::kNoSpeechTimeoutMs = 8000;
 108 const int SpeechRecognizerImpl::kEndpointerEstimationTimeMs = 300;
 109 media::AudioManager* SpeechRecognizerImpl::audio_manager_for_tests_ = NULL;
 110
 111 COMPILE_ASSERT(SpeechRecognizerImpl::kNumBitsPerAudioSample % 8 == 0,
 112                kNumBitsPerAudioSample_must_be_a_multiple_of_8);
 113
 114 // SpeechRecognizerImpl::OnDataConverter implementation
 115
 116 SpeechRecognizerImpl::OnDataConverter::OnDataConverter(
 117     const AudioParameters& input_params, const AudioParameters& output_params)
 118     : audio_converter_(input_params, output_params, false),
 119       input_bus_(AudioBus::Create(input_params)),
 120       output_bus_(AudioBus::Create(output_params)),
 121       input_parameters_(input_params),
 122       output_parameters_(output_params),
 123       waiting_for_input_(false),
 124       converted_data_(new uint8[output_parameters_.GetBytesPerBuffer()]) {
 125   audio_converter_.AddInput(this);
 126 }
 127
 128 SpeechRecognizerImpl::OnDataConverter::~OnDataConverter() {
 129   // It should now be safe to unregister the converter since no more OnData()
 130   // callbacks are outstanding at this point.
 131   audio_converter_.RemoveInput(this);
 132 }
 133
 134 scoped_refptr<AudioChunk> SpeechRecognizerImpl::OnDataConverter::Convert(
 135     const AudioBus* data) {
 136   CHECK_EQ(data->frames(), input_parameters_.frames_per_buffer());
 137
 138   data->CopyTo(input_bus_.get());
 139
 140   waiting_for_input_ = true;
 141   audio_converter_.Convert(output_bus_.get());
 142
 143   output_bus_->ToInterleaved(
 144       output_bus_->frames(), output_parameters_.bits_per_sample() / 8,
 145       converted_data_.get());
 146
 147   // TODO(primiano): Refactor AudioChunk to avoid the extra-copy here
 148   // (see http://crbug.com/249316 for details).
 149   return scoped_refptr<AudioChunk>(new AudioChunk(
 150       converted_data_.get(),
 151       output_parameters_.GetBytesPerBuffer(),
 152       output_parameters_.bits_per_sample() / 8));
 153 }
 154
 155 double SpeechRecognizerImpl::OnDataConverter::ProvideInput(
 156     AudioBus* dest, base::TimeDelta buffer_delay) {
 157   // The audio converted should never ask for more than one bus in each call
 158   // to Convert(). If so, we have a serious issue in our design since we might
 159   // miss recorded chunks of 100 ms audio data.
 160   CHECK(waiting_for_input_);
 161
 162   // Read from the input bus to feed the converter.
 163   input_bus_->CopyTo(dest);
 164
 165   // |input_bus_| should only be provide once.
 166   waiting_for_input_ = false;
 167   return 1;
 168 }
 169
 170 // SpeechRecognizerImpl implementation
 171
 172 SpeechRecognizerImpl::SpeechRecognizerImpl(
 173     SpeechRecognitionEventListener* listener,
 174     int session_id,
 175     bool continuous,
 176     bool provisional_results,
 177     SpeechRecognitionEngine* engine)
 178     : SpeechRecognizer(listener, session_id),
 179       recognition_engine_(engine),
 180       endpointer_(kAudioSampleRate),
 181       audio_log_(MediaInternals::GetInstance()->CreateAudioLog(
 182           media::AudioLogFactory::AUDIO_INPUT_CONTROLLER)),
 183       is_dispatching_event_(false),
 184       provisional_results_(provisional_results),
 185       state_(STATE_IDLE) {
 186   DCHECK(recognition_engine_ != NULL);
 187   if (!continuous) {
 188     // In single shot (non-continous) recognition,
 189     // the session is automatically ended after:
 190     //  - 0.5 seconds of silence if time <  3 seconds
 191     //  - 1   seconds of silence if time >= 3 seconds
 192     endpointer_.set_speech_input_complete_silence_length(
 193         base::Time::kMicrosecondsPerSecond / 2);
 194     endpointer_.set_long_speech_input_complete_silence_length(
 195         base::Time::kMicrosecondsPerSecond);
 196     endpointer_.set_long_speech_length(3 * base::Time::kMicrosecondsPerSecond);
 197   } else {
 198     // In continuous recognition, the session is automatically ended after 15
 199     // seconds of silence.
 200     const int64 cont_timeout_us = base::Time::kMicrosecondsPerSecond * 15;
 201     endpointer_.set_speech_input_complete_silence_length(cont_timeout_us);
 202     endpointer_.set_long_speech_length(0);  // Use only a single timeout.
 203   }
 204   endpointer_.StartSession();
 205   recognition_engine_->set_delegate(this);
 206 }
 207
 208 // -------  Methods that trigger Finite State Machine (FSM) events ------------
 209
 210 // NOTE:all the external events and requests should be enqueued (PostTask), even
 211 // if they come from the same (IO) thread, in order to preserve the relationship
 212 // of causality between events and avoid interleaved event processing due to
 213 // synchronous callbacks.
 214
 215 void SpeechRecognizerImpl::StartRecognition(const std::string& device_id) {
 216   DCHECK(!device_id.empty());
 217   device_id_ = device_id;
 218
 219   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 220                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 221                                      this, FSMEventArgs(EVENT_START)));
 222 }
 223
 224 void SpeechRecognizerImpl::AbortRecognition() {
 225   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 226                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 227                                      this, FSMEventArgs(EVENT_ABORT)));
 228 }
 229
 230 void SpeechRecognizerImpl::StopAudioCapture() {
 231   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 232                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 233                                      this, FSMEventArgs(EVENT_STOP_CAPTURE)));
 234 }
 235
 236 bool SpeechRecognizerImpl::IsActive() const {
 237   // Checking the FSM state from another thread (thus, while the FSM is
 238   // potentially concurrently evolving) is meaningless.
 239   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 240   return state_ != STATE_IDLE && state_ != STATE_ENDED;
 241 }
 242
 243 bool SpeechRecognizerImpl::IsCapturingAudio() const {
 244   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); // See IsActive().
 245   const bool is_capturing_audio = state_ >= STATE_STARTING &&
 246                                   state_ <= STATE_RECOGNIZING;
 247   DCHECK((is_capturing_audio && (audio_controller_.get() != NULL)) ||
 248          (!is_capturing_audio && audio_controller_.get() == NULL));
 249   return is_capturing_audio;
 250 }
 251
 252 const SpeechRecognitionEngine&
 253 SpeechRecognizerImpl::recognition_engine() const {
 254   return *(recognition_engine_.get());
 255 }
 256
 257 SpeechRecognizerImpl::~SpeechRecognizerImpl() {
 258   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 259   endpointer_.EndSession();
 260   if (audio_controller_.get()) {
 261     audio_controller_->Close(
 262         base::Bind(&KeepAudioControllerRefcountedForDtor, audio_controller_));
 263     audio_log_->OnClosed(0);
 264   }
 265 }
 266
 267 // Invoked in the audio thread.
 268 void SpeechRecognizerImpl::OnError(AudioInputController* controller,
 269     media::AudioInputController::ErrorCode error_code) {
 270   FSMEventArgs event_args(EVENT_AUDIO_ERROR);
 271   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 272                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 273                                      this, event_args));
 274 }
 275
 276 void SpeechRecognizerImpl::OnData(AudioInputController* controller,
 277                                   const AudioBus* data) {
 278   // Convert audio from native format to fixed format used by WebSpeech.
 279   FSMEventArgs event_args(EVENT_AUDIO_DATA);
 280   event_args.audio_data = audio_converter_->Convert(data);
 281
 282   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 283                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 284                                      this, event_args));
 285 }
 286
 287 void SpeechRecognizerImpl::OnAudioClosed(AudioInputController*) {}
 288
 289 void SpeechRecognizerImpl::OnSpeechRecognitionEngineResults(
 290     const SpeechRecognitionResults& results) {
 291   FSMEventArgs event_args(EVENT_ENGINE_RESULT);
 292   event_args.engine_results = results;
 293   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 294                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 295                                      this, event_args));
 296 }
 297
 298 void SpeechRecognizerImpl::OnSpeechRecognitionEngineError(
 299     const SpeechRecognitionError& error) {
 300   FSMEventArgs event_args(EVENT_ENGINE_ERROR);
 301   event_args.engine_error = error;
 302   BrowserThread::PostTask(BrowserThread::IO, FROM_HERE,
 303                           base::Bind(&SpeechRecognizerImpl::DispatchEvent,
 304                                      this, event_args));
 305 }
 306
 307 // -----------------------  Core FSM implementation ---------------------------
 308 // TODO(primiano): After the changes in the media package (r129173), this class
 309 // slightly violates the SpeechRecognitionEventListener interface contract. In
 310 // particular, it is not true anymore that this class can be freed after the
 311 // OnRecognitionEnd event, since the audio_controller_.Close() asynchronous
 312 // call can be still in progress after the end event. Currently, it does not
 313 // represent a problem for the browser itself, since refcounting protects us
 314 // against such race conditions. However, we should fix this in the next CLs.
 315 // For instance, tests are currently working just because the
 316 // TestAudioInputController is not closing asynchronously as the real controller
 317 // does, but they will become flaky if TestAudioInputController will be fixed.
 318
 319 void SpeechRecognizerImpl::DispatchEvent(const FSMEventArgs& event_args) {
 320   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 321   DCHECK_LE(event_args.event, EVENT_MAX_VALUE);
 322   DCHECK_LE(state_, STATE_MAX_VALUE);
 323
 324   // Event dispatching must be sequential, otherwise it will break all the rules
 325   // and the assumptions of the finite state automata model.
 326   DCHECK(!is_dispatching_event_);
 327   is_dispatching_event_ = true;
 328
 329   // Guard against the delegate freeing us until we finish processing the event.
 330   scoped_refptr<SpeechRecognizerImpl> me(this);
 331
 332   if (event_args.event == EVENT_AUDIO_DATA) {
 333     DCHECK(event_args.audio_data.get() != NULL);
 334     ProcessAudioPipeline(*event_args.audio_data.get());
 335   }
 336
 337   // The audio pipeline must be processed before the event dispatch, otherwise
 338   // it would take actions according to the future state instead of the current.
 339   state_ = ExecuteTransitionAndGetNextState(event_args);
 340   is_dispatching_event_ = false;
 341 }
 342
 343 SpeechRecognizerImpl::FSMState
 344 SpeechRecognizerImpl::ExecuteTransitionAndGetNextState(
 345     const FSMEventArgs& event_args) {
 346   const FSMEvent event = event_args.event;
 347   switch (state_) {
 348     case STATE_IDLE:
 349       switch (event) {
 350         // TODO(primiano): restore UNREACHABLE_CONDITION on EVENT_ABORT and
 351         // EVENT_STOP_CAPTURE below once speech input extensions are fixed.
 352         case EVENT_ABORT:
 353           return AbortSilently(event_args);
 354         case EVENT_START:
 355           return StartRecording(event_args);
 356         case EVENT_STOP_CAPTURE:
 357           return AbortSilently(event_args);
 358         case EVENT_AUDIO_DATA:     // Corner cases related to queued messages
 359         case EVENT_ENGINE_RESULT:  // being lately dispatched.
 360         case EVENT_ENGINE_ERROR:
 361         case EVENT_AUDIO_ERROR:
 362           return DoNothing(event_args);
 363       }
 364       break;
 365     case STATE_STARTING:
 366       switch (event) {
 367         case EVENT_ABORT:
 368           return AbortWithError(event_args);
 369         case EVENT_START:
 370           return NotFeasible(event_args);
 371         case EVENT_STOP_CAPTURE:
 372           return AbortSilently(event_args);
 373         case EVENT_AUDIO_DATA:
 374           return StartRecognitionEngine(event_args);
 375         case EVENT_ENGINE_RESULT:
 376           return NotFeasible(event_args);
 377         case EVENT_ENGINE_ERROR:
 378         case EVENT_AUDIO_ERROR:
 379           return AbortWithError(event_args);
 380       }
 381       break;
 382     case STATE_ESTIMATING_ENVIRONMENT:
 383       switch (event) {
 384         case EVENT_ABORT:
 385           return AbortWithError(event_args);
 386         case EVENT_START:
 387           return NotFeasible(event_args);
 388         case EVENT_STOP_CAPTURE:
 389           return StopCaptureAndWaitForResult(event_args);
 390         case EVENT_AUDIO_DATA:
 391           return WaitEnvironmentEstimationCompletion(event_args);
 392         case EVENT_ENGINE_RESULT:
 393           return ProcessIntermediateResult(event_args);
 394         case EVENT_ENGINE_ERROR:
 395         case EVENT_AUDIO_ERROR:
 396           return AbortWithError(event_args);
 397       }
 398       break;
 399     case STATE_WAITING_FOR_SPEECH:
 400       switch (event) {
 401         case EVENT_ABORT:
 402           return AbortWithError(event_args);
 403         case EVENT_START:
 404           return NotFeasible(event_args);
 405         case EVENT_STOP_CAPTURE:
 406           return StopCaptureAndWaitForResult(event_args);
 407         case EVENT_AUDIO_DATA:
 408           return DetectUserSpeechOrTimeout(event_args);
 409         case EVENT_ENGINE_RESULT:
 410           return ProcessIntermediateResult(event_args);
 411         case EVENT_ENGINE_ERROR:
 412         case EVENT_AUDIO_ERROR:
 413           return AbortWithError(event_args);
 414       }
 415       break;
 416     case STATE_RECOGNIZING:
 417       switch (event) {
 418         case EVENT_ABORT:
 419           return AbortWithError(event_args);
 420         case EVENT_START:
 421           return NotFeasible(event_args);
 422         case EVENT_STOP_CAPTURE:
 423           return StopCaptureAndWaitForResult(event_args);
 424         case EVENT_AUDIO_DATA:
 425           return DetectEndOfSpeech(event_args);
 426         case EVENT_ENGINE_RESULT:
 427           return ProcessIntermediateResult(event_args);
 428         case EVENT_ENGINE_ERROR:
 429         case EVENT_AUDIO_ERROR:
 430           return AbortWithError(event_args);
 431       }
 432       break;
 433     case STATE_WAITING_FINAL_RESULT:
 434       switch (event) {
 435         case EVENT_ABORT:
 436           return AbortWithError(event_args);
 437         case EVENT_START:
 438           return NotFeasible(event_args);
 439         case EVENT_STOP_CAPTURE:
 440         case EVENT_AUDIO_DATA:
 441           return DoNothing(event_args);
 442         case EVENT_ENGINE_RESULT:
 443           return ProcessFinalResult(event_args);
 444         case EVENT_ENGINE_ERROR:
 445         case EVENT_AUDIO_ERROR:
 446           return AbortWithError(event_args);
 447       }
 448       break;
 449
 450     // TODO(primiano): remove this state when speech input extensions support
 451     // will be removed and STATE_IDLE.EVENT_ABORT,EVENT_STOP_CAPTURE will be
 452     // reset to NotFeasible (see TODO above).
 453     case STATE_ENDED:
 454       return DoNothing(event_args);
 455   }
 456   return NotFeasible(event_args);
 457 }
 458
 459 // ----------- Contract for all the FSM evolution functions below -------------
 460 //  - Are guaranteed to be executed in the IO thread;
 461 //  - Are guaranteed to be not reentrant (themselves and each other);
 462 //  - event_args members are guaranteed to be stable during the call;
 463 //  - The class won't be freed in the meanwhile due to callbacks;
 464 //  - IsCapturingAudio() returns true if and only if audio_controller_ != NULL.
 465
 466 // TODO(primiano): the audio pipeline is currently serial. However, the
 467 // clipper->endpointer->vumeter chain and the sr_engine could be parallelized.
 468 // We should profile the execution to see if it would be worth or not.
 469 void SpeechRecognizerImpl::ProcessAudioPipeline(const AudioChunk& raw_audio) {
 470   const bool route_to_endpointer = state_ >= STATE_ESTIMATING_ENVIRONMENT &&
 471                                    state_ <= STATE_RECOGNIZING;
 472   const bool route_to_sr_engine = route_to_endpointer;
 473   const bool route_to_vumeter = state_ >= STATE_WAITING_FOR_SPEECH &&
 474                                 state_ <= STATE_RECOGNIZING;
 475   const bool clip_detected = DetectClipping(raw_audio);
 476   float rms = 0.0f;
 477
 478   num_samples_recorded_ += raw_audio.NumSamples();
 479
 480   if (route_to_endpointer)
 481     endpointer_.ProcessAudio(raw_audio, &rms);
 482
 483   if (route_to_vumeter) {
 484     DCHECK(route_to_endpointer); // Depends on endpointer due to |rms|.
 485     UpdateSignalAndNoiseLevels(rms, clip_detected);
 486   }
 487   if (route_to_sr_engine) {
 488     DCHECK(recognition_engine_.get() != NULL);
 489     recognition_engine_->TakeAudioChunk(raw_audio);
 490   }
 491 }
 492
 493 SpeechRecognizerImpl::FSMState
 494 SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
 495   DCHECK(recognition_engine_.get() != NULL);
 496   DCHECK(!IsCapturingAudio());
 497   const bool unit_test_is_active = (audio_manager_for_tests_ != NULL);
 498   AudioManager* audio_manager = unit_test_is_active ?
 499                                 audio_manager_for_tests_ :
 500                                 AudioManager::Get();
 501   DCHECK(audio_manager != NULL);
 502
 503   DVLOG(1) << "SpeechRecognizerImpl starting audio capture.";
 504   num_samples_recorded_ = 0;
 505   audio_level_ = 0;
 506   listener()->OnRecognitionStart(session_id());
 507
 508   // TODO(xians): Check if the OS has the device with |device_id_|, return
 509   // |SPEECH_AUDIO_ERROR_DETAILS_NO_MIC| if the target device does not exist.
 510   if (!audio_manager->HasAudioInputDevices()) {
 511     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO,
 512                                         SPEECH_AUDIO_ERROR_DETAILS_NO_MIC));
 513   }
 514
 515   int chunk_duration_ms = recognition_engine_->GetDesiredAudioChunkDurationMs();
 516
 517   AudioParameters in_params = audio_manager->GetInputStreamParameters(
 518       device_id_);
 519   if (!in_params.IsValid() && !unit_test_is_active) {
 520     DLOG(ERROR) << "Invalid native audio input parameters";
 521     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
 522   }
 523
 524   // Audio converter shall provide audio based on these parameters as output.
 525   // Hard coded, WebSpeech specific parameters are utilized here.
 526   int frames_per_buffer = (kAudioSampleRate * chunk_duration_ms) / 1000;
 527   AudioParameters output_parameters = AudioParameters(
 528       AudioParameters::AUDIO_PCM_LOW_LATENCY, kChannelLayout, kAudioSampleRate,
 529       kNumBitsPerAudioSample, frames_per_buffer);
 530
 531   // Audio converter will receive audio based on these parameters as input.
 532   // On Windows we start by verifying that Core Audio is supported. If not,
 533   // the WaveIn API is used and we might as well avoid all audio conversations
 534   // since WaveIn does the conversion for us.
 535   // TODO(henrika): this code should be moved to platform dependent audio
 536   // managers.
 537   bool use_native_audio_params = true;
 538 #if defined(OS_WIN)
 539   use_native_audio_params = media::CoreAudioUtil::IsSupported();
 540   DVLOG_IF(1, !use_native_audio_params) << "Reverting to WaveIn for WebSpeech";
 541 #endif
 542
 543   AudioParameters input_parameters = output_parameters;
 544   if (use_native_audio_params && !unit_test_is_active) {
 545     // Use native audio parameters but avoid opening up at the native buffer
 546     // size. Instead use same frame size (in milliseconds) as WebSpeech uses.
 547     // We rely on internal buffers in the audio back-end to fulfill this request
 548     // and the idea is to simplify the audio conversion since each Convert()
 549     // call will then render exactly one ProvideInput() call.
 550     // Due to implementation details in the audio converter, 2 milliseconds
 551     // are added to the default frame size (100 ms) to ensure there is enough
 552     // data to generate 100 ms of output when resampling.
 553     frames_per_buffer =
 554         ((in_params.sample_rate() * (chunk_duration_ms + 2)) / 1000.0) + 0.5;
 555     input_parameters.Reset(in_params.format(),
 556                            in_params.channel_layout(),
 557                            in_params.channels(),
 558                            in_params.sample_rate(),
 559                            in_params.bits_per_sample(),
 560                            frames_per_buffer);
 561   }
 562
 563   // Create an audio converter which converts data between native input format
 564   // and WebSpeech specific output format.
 565   audio_converter_.reset(
 566       new OnDataConverter(input_parameters, output_parameters));
 567
 568   audio_controller_ = AudioInputController::Create(
 569       audio_manager, this, input_parameters, device_id_, NULL);
 570
 571   if (!audio_controller_.get()) {
 572     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
 573   }
 574
 575   audio_log_->OnCreated(0, input_parameters, device_id_);
 576
 577   // The endpointer needs to estimate the environment/background noise before
 578   // starting to treat the audio as user input. We wait in the state
 579   // ESTIMATING_ENVIRONMENT until such interval has elapsed before switching
 580   // to user input mode.
 581   endpointer_.SetEnvironmentEstimationMode();
 582   audio_controller_->Record();
 583   audio_log_->OnStarted(0);
 584   return STATE_STARTING;
 585 }
 586
 587 SpeechRecognizerImpl::FSMState
 588 SpeechRecognizerImpl::StartRecognitionEngine(const FSMEventArgs& event_args) {
 589   // This is the first audio packet captured, so the recognition engine is
 590   // started and the delegate notified about the event.
 591   DCHECK(recognition_engine_.get() != NULL);
 592   recognition_engine_->StartRecognition();
 593   listener()->OnAudioStart(session_id());
 594
 595   // This is a little hack, since TakeAudioChunk() is already called by
 596   // ProcessAudioPipeline(). It is the best tradeoff, unless we allow dropping
 597   // the first audio chunk captured after opening the audio device.
 598   recognition_engine_->TakeAudioChunk(*(event_args.audio_data.get()));
 599   return STATE_ESTIMATING_ENVIRONMENT;
 600 }
 601
 602 SpeechRecognizerImpl::FSMState
 603 SpeechRecognizerImpl::WaitEnvironmentEstimationCompletion(const FSMEventArgs&) {
 604   DCHECK(endpointer_.IsEstimatingEnvironment());
 605   if (GetElapsedTimeMs() >= kEndpointerEstimationTimeMs) {
 606     endpointer_.SetUserInputMode();
 607     listener()->OnEnvironmentEstimationComplete(session_id());
 608     return STATE_WAITING_FOR_SPEECH;
 609   } else {
 610     return STATE_ESTIMATING_ENVIRONMENT;
 611   }
 612 }
 613
 614 SpeechRecognizerImpl::FSMState
 615 SpeechRecognizerImpl::DetectUserSpeechOrTimeout(const FSMEventArgs&) {
 616   if (endpointer_.DidStartReceivingSpeech()) {
 617     listener()->OnSoundStart(session_id());
 618     return STATE_RECOGNIZING;
 619   } else if (GetElapsedTimeMs() >= kNoSpeechTimeoutMs) {
 620     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NO_SPEECH));
 621   }
 622   return STATE_WAITING_FOR_SPEECH;
 623 }
 624
 625 SpeechRecognizerImpl::FSMState
 626 SpeechRecognizerImpl::DetectEndOfSpeech(const FSMEventArgs& event_args) {
 627   if (endpointer_.speech_input_complete())
 628     return StopCaptureAndWaitForResult(event_args);
 629   return STATE_RECOGNIZING;
 630 }
 631
 632 SpeechRecognizerImpl::FSMState
 633 SpeechRecognizerImpl::StopCaptureAndWaitForResult(const FSMEventArgs&) {
 634   DCHECK(state_ >= STATE_ESTIMATING_ENVIRONMENT && state_ <= STATE_RECOGNIZING);
 635
 636   DVLOG(1) << "Concluding recognition";
 637   CloseAudioControllerAsynchronously();
 638   recognition_engine_->AudioChunksEnded();
 639
 640   if (state_ > STATE_WAITING_FOR_SPEECH)
 641     listener()->OnSoundEnd(session_id());
 642
 643   listener()->OnAudioEnd(session_id());
 644   return STATE_WAITING_FINAL_RESULT;
 645 }
 646
 647 SpeechRecognizerImpl::FSMState
 648 SpeechRecognizerImpl::AbortSilently(const FSMEventArgs& event_args) {
 649   DCHECK_NE(event_args.event, EVENT_AUDIO_ERROR);
 650   DCHECK_NE(event_args.event, EVENT_ENGINE_ERROR);
 651   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_NONE));
 652 }
 653
 654 SpeechRecognizerImpl::FSMState
 655 SpeechRecognizerImpl::AbortWithError(const FSMEventArgs& event_args) {
 656   if (event_args.event == EVENT_AUDIO_ERROR) {
 657     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));
 658   } else if (event_args.event == EVENT_ENGINE_ERROR) {
 659     return Abort(event_args.engine_error);
 660   }
 661   return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_ABORTED));
 662 }
 663
 664 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::Abort(
 665     const SpeechRecognitionError& error) {
 666   if (IsCapturingAudio())
 667     CloseAudioControllerAsynchronously();
 668
 669   DVLOG(1) << "SpeechRecognizerImpl canceling recognition. ";
 670
 671   // The recognition engine is initialized only after STATE_STARTING.
 672   if (state_ > STATE_STARTING) {
 673     DCHECK(recognition_engine_.get() != NULL);
 674     recognition_engine_->EndRecognition();
 675   }
 676
 677   if (state_ > STATE_WAITING_FOR_SPEECH && state_ < STATE_WAITING_FINAL_RESULT)
 678     listener()->OnSoundEnd(session_id());
 679
 680   if (state_ > STATE_STARTING && state_ < STATE_WAITING_FINAL_RESULT)
 681     listener()->OnAudioEnd(session_id());
 682
 683   if (error.code != SPEECH_RECOGNITION_ERROR_NONE)
 684     listener()->OnRecognitionError(session_id(), error);
 685
 686   listener()->OnRecognitionEnd(session_id());
 687
 688   return STATE_ENDED;
 689 }
 690
 691 SpeechRecognizerImpl::FSMState SpeechRecognizerImpl::ProcessIntermediateResult(
 692     const FSMEventArgs& event_args) {
 693   // Provisional results can occur only if explicitly enabled in the JS API.
 694   DCHECK(provisional_results_);
 695
 696   // In continuous recognition, intermediate results can occur even when we are
 697   // in the ESTIMATING_ENVIRONMENT or WAITING_FOR_SPEECH states (if the
 698   // recognition engine is "faster" than our endpointer). In these cases we
 699   // skip the endpointer and fast-forward to the RECOGNIZING state, with respect
 700   // of the events triggering order.
 701   if (state_ == STATE_ESTIMATING_ENVIRONMENT) {
 702     DCHECK(endpointer_.IsEstimatingEnvironment());
 703     endpointer_.SetUserInputMode();
 704     listener()->OnEnvironmentEstimationComplete(session_id());
 705   } else if (state_ == STATE_WAITING_FOR_SPEECH) {
 706     listener()->OnSoundStart(session_id());
 707   } else {
 708     DCHECK_EQ(STATE_RECOGNIZING, state_);
 709   }
 710
 711   listener()->OnRecognitionResults(session_id(), event_args.engine_results);
 712   return STATE_RECOGNIZING;
 713 }
 714
 715 SpeechRecognizerImpl::FSMState
 716 SpeechRecognizerImpl::ProcessFinalResult(const FSMEventArgs& event_args) {
 717   const SpeechRecognitionResults& results = event_args.engine_results;
 718   SpeechRecognitionResults::const_iterator i = results.begin();
 719   bool provisional_results_pending = false;
 720   bool results_are_empty = true;
 721   for (; i != results.end(); ++i) {
 722     const SpeechRecognitionResult& result = *i;
 723     if (result.is_provisional) {
 724       DCHECK(provisional_results_);
 725       provisional_results_pending = true;
 726     } else if (results_are_empty) {
 727       results_are_empty = result.hypotheses.empty();
 728     }
 729   }
 730
 731   if (provisional_results_pending) {
 732     listener()->OnRecognitionResults(session_id(), results);
 733     // We don't end the recognition if a provisional result is received in
 734     // STATE_WAITING_FINAL_RESULT. A definitive result will come next and will
 735     // end the recognition.
 736     return state_;
 737   }
 738
 739   recognition_engine_->EndRecognition();
 740
 741   if (!results_are_empty) {
 742     // We could receive an empty result (which we won't propagate further)
 743     // in the following (continuous) scenario:
 744     //  1. The caller start pushing audio and receives some results;
 745     //  2. A |StopAudioCapture| is issued later;
 746     //  3. The final audio frames captured in the interval ]1,2] do not lead to
 747     //     any result (nor any error);
 748     //  4. The speech recognition engine, therefore, emits an empty result to
 749     //     notify that the recognition is ended with no error, yet neither any
 750     //     further result.
 751     listener()->OnRecognitionResults(session_id(), results);
 752   }
 753
 754   listener()->OnRecognitionEnd(session_id());
 755   return STATE_ENDED;
 756 }
 757
 758 SpeechRecognizerImpl::FSMState
 759 SpeechRecognizerImpl::DoNothing(const FSMEventArgs&) const {
 760   return state_;  // Just keep the current state.
 761 }
 762
 763 SpeechRecognizerImpl::FSMState
 764 SpeechRecognizerImpl::NotFeasible(const FSMEventArgs& event_args) {
 765   NOTREACHED() << "Unfeasible event " << event_args.event
 766                << " in state " << state_;
 767   return state_;
 768 }
 769
 770 void SpeechRecognizerImpl::CloseAudioControllerAsynchronously() {
 771   DCHECK(IsCapturingAudio());
 772   DVLOG(1) << "SpeechRecognizerImpl closing audio controller.";
 773   // Issues a Close on the audio controller, passing an empty callback. The only
 774   // purpose of such callback is to keep the audio controller refcounted until
 775   // Close has completed (in the audio thread) and automatically destroy it
 776   // afterwards (upon return from OnAudioClosed).
 777   audio_controller_->Close(base::Bind(&SpeechRecognizerImpl::OnAudioClosed,
 778                                       this, audio_controller_));
 779   audio_controller_ = NULL;  // The controller is still refcounted by Bind.
 780   audio_log_->OnClosed(0);
 781 }
 782
 783 int SpeechRecognizerImpl::GetElapsedTimeMs() const {
 784   return (num_samples_recorded_ * 1000) / kAudioSampleRate;
 785 }
 786
 787 void SpeechRecognizerImpl::UpdateSignalAndNoiseLevels(const float& rms,
 788                                                   bool clip_detected) {
 789   // Calculate the input volume to display in the UI, smoothing towards the
 790   // new level.
 791   // TODO(primiano): Do we really need all this floating point arith here?
 792   // Perhaps it might be quite expensive on mobile.
 793   float level = (rms - kAudioMeterMinDb) /
 794       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
 795   level = std::min(std::max(0.0f, level), kAudioMeterRangeMaxUnclipped);
 796   const float smoothing_factor = (level > audio_level_) ? kUpSmoothingFactor :
 797                                                           kDownSmoothingFactor;
 798   audio_level_ += (level - audio_level_) * smoothing_factor;
 799
 800   float noise_level = (endpointer_.NoiseLevelDb() - kAudioMeterMinDb) /
 801       (kAudioMeterDbRange / kAudioMeterRangeMaxUnclipped);
 802   noise_level = std::min(std::max(0.0f, noise_level),
 803                          kAudioMeterRangeMaxUnclipped);
 804
 805   listener()->OnAudioLevelsChange(
 806       session_id(), clip_detected ? 1.0f : audio_level_, noise_level);
 807 }
 808
 809 void SpeechRecognizerImpl::SetAudioManagerForTesting(
 810     AudioManager* audio_manager) {
 811   audio_manager_for_tests_ = audio_manager;
 812 }
 813
 814 SpeechRecognizerImpl::FSMEventArgs::FSMEventArgs(FSMEvent event_value)
 815     : event(event_value),
 816       audio_data(NULL),
 817       engine_error(SPEECH_RECOGNITION_ERROR_NONE) {
 818 }
 819
 820 SpeechRecognizerImpl::FSMEventArgs::~FSMEventArgs() {
 821 }
 822
 823 }  // namespace content