chrome/browser/speech/speech_recognition_bubble.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/speech/speech_recognition_bubble.h"
   6
   7 #include "base/bind.h"
   8 #include "base/lazy_instance.h"
   9 #include "base/message_loop/message_loop.h"
  10 #include "content/public/browser/web_contents.h"
  11 #include "content/public/browser/web_contents_view.h"
  12 #include "grit/generated_resources.h"
  13 #include "grit/theme_resources.h"
  14 #include "ui/base/resource/resource_bundle.h"
  15 #include "ui/gfx/canvas.h"
  16 #include "ui/gfx/display.h"
  17 #include "ui/gfx/image/image_skia_operations.h"
  18 #include "ui/gfx/rect.h"
  19 #include "ui/gfx/screen.h"
  20
  21 using content::WebContents;
  22
  23 namespace {
  24
  25 const color_utils::HSL kGrayscaleShift = { -1, 0, 0.6 };
  26 const int kWarmingUpAnimationStartMs = 500;
  27 const int kWarmingUpAnimationStepMs = 100;
  28 const int kRecognizingAnimationStepMs = 100;
  29
  30 // A lazily initialized singleton to hold all the image used by the speech
  31 // recognition bubbles and safely destroy them on exit.
  32 class SpeechRecognitionBubbleImages {
  33  public:
  34   const std::vector<gfx::ImageSkia>& spinner() const { return spinner_; }
  35   const std::vector<gfx::ImageSkia>& warm_up() const { return warm_up_; }
  36   gfx::ImageSkia* mic_full() const { return mic_full_; }
  37   gfx::ImageSkia* mic_empty() const { return mic_empty_; }
  38   gfx::ImageSkia* mic_noise() const { return mic_noise_; }
  39   gfx::ImageSkia* mic_mask() const { return mic_mask_; }
  40
  41  private:
  42   // Private constructor to enforce singleton.
  43   friend struct base::DefaultLazyInstanceTraits<SpeechRecognitionBubbleImages>;
  44   SpeechRecognitionBubbleImages();
  45
  46   std::vector<gfx::ImageSkia> spinner_;  // Frames for the progress spinner.
  47   std::vector<gfx::ImageSkia> warm_up_;  // Frames for the warm up animation.
  48
  49   // These images are owned by ResourceBundle and need not be destroyed.
  50   gfx::ImageSkia* mic_full_;  // Mic image with full volume.
  51   gfx::ImageSkia* mic_noise_;  // Mic image with full noise volume.
  52   gfx::ImageSkia* mic_empty_;  // Mic image with zero volume.
  53   gfx::ImageSkia* mic_mask_;  // Gradient mask used by the volume indicator.
  54 };
  55
  56 SpeechRecognitionBubbleImages::SpeechRecognitionBubbleImages() {
  57   mic_empty_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  58       IDR_SPEECH_INPUT_MIC_EMPTY);
  59   mic_noise_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  60       IDR_SPEECH_INPUT_MIC_NOISE);
  61   mic_full_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  62       IDR_SPEECH_INPUT_MIC_FULL);
  63   mic_mask_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  64       IDR_SPEECH_INPUT_MIC_MASK);
  65
  66   // The sprite image consists of all the animation frames put together in one
  67   // horizontal/wide image. Each animation frame is square in shape within the
  68   // sprite.
  69   const gfx::ImageSkia* spinner_image = ui::ResourceBundle::GetSharedInstance().
  70       GetImageSkiaNamed(IDR_SPEECH_INPUT_SPINNER);
  71   int frame_size = spinner_image->height();
  72
  73   // When recording starts up, it may take a short while (few ms or even a
  74   // couple of seconds) before the audio device starts really capturing data.
  75   // This is more apparent on first use. To cover such cases we show a warming
  76   // up state in the bubble starting with a blank spinner image. If audio data
  77   // starts coming in within a couple hundred ms, we switch to the recording
  78   // UI and if it takes longer, we show the real warm up animation frames.
  79   // This reduces visual jank for the most part.
  80   SkBitmap empty_spinner;
  81   empty_spinner.setConfig(SkBitmap::kARGB_8888_Config, frame_size, frame_size);
  82   empty_spinner.allocPixels();
  83   empty_spinner.eraseRGB(255, 255, 255);
  84   // |empty_spinner| has solid color. Pixel doubling a solid color is ok.
  85   warm_up_.push_back(gfx::ImageSkia::CreateFrom1xBitmap(empty_spinner));
  86
  87   for (gfx::Rect src_rect(frame_size, frame_size);
  88        src_rect.x() < spinner_image->width();
  89        src_rect.Offset(frame_size, 0)) {
  90     gfx::ImageSkia frame = gfx::ImageSkiaOperations::ExtractSubset(
  91         *spinner_image, src_rect);
  92
  93     // The image created by ExtractSubset just points to the same pixels as
  94     // the original and adjusts rowBytes accordingly. However that doesn't
  95     // render properly and gets vertically squished in Linux due to a bug in
  96     // Skia. Until that gets fixed we work around by taking a real copy of it
  97     // below as the copied image has the correct rowBytes and renders fine.
  98     frame.EnsureRepsForSupportedScales();
  99     std::vector<gfx::ImageSkiaRep> image_reps = frame.image_reps();
 100     gfx::ImageSkia frame_copy;
 101     for (size_t i = 0; i < image_reps.size(); ++i) {
 102       const SkBitmap& copy_src = image_reps[i].sk_bitmap();
 103       SkBitmap copy_dst;
 104       copy_src.copyTo(&copy_dst, SkBitmap::kARGB_8888_Config);
 105       frame_copy.AddRepresentation(gfx::ImageSkiaRep(
 106           copy_dst, image_reps[i].scale()));
 107     }
 108     spinner_.push_back(frame_copy);
 109
 110     // The warm up spinner animation is a gray scale version of the real one.
 111     warm_up_.push_back(gfx::ImageSkiaOperations::CreateHSLShiftedImage(
 112         frame_copy, kGrayscaleShift));
 113   }
 114 }
 115
 116 base::LazyInstance<SpeechRecognitionBubbleImages> g_images =
 117     LAZY_INSTANCE_INITIALIZER;
 118
 119 }  // namespace
 120
 121 SpeechRecognitionBubble::FactoryMethod SpeechRecognitionBubble::factory_ = NULL;
 122 const int SpeechRecognitionBubble::kBubbleTargetOffsetX = 10;
 123
 124 SpeechRecognitionBubble* SpeechRecognitionBubble::Create(
 125     WebContents* web_contents, Delegate* delegate,
 126     const gfx::Rect& element_rect) {
 127   if (factory_)
 128     return (*factory_)(web_contents, delegate, element_rect);
 129
 130   // Has the tab already closed before bubble create request was processed?
 131   if (!web_contents)
 132     return NULL;
 133
 134   return CreateNativeBubble(web_contents, delegate, element_rect);
 135 }
 136
 137 SpeechRecognitionBubbleBase::SpeechRecognitionBubbleBase(
 138     WebContents* web_contents)
 139     : weak_factory_(this),
 140       animation_step_(0),
 141       display_mode_(DISPLAY_MODE_RECORDING),
 142       web_contents_(web_contents),
 143       scale_(1.0f) {
 144   gfx::NativeView view =
 145       web_contents_ ? web_contents_->GetView()->GetNativeView() : NULL;
 146   gfx::Screen* screen = gfx::Screen::GetScreenFor(view);
 147   gfx::Display display = screen->GetDisplayNearestWindow(view);
 148   scale_ = display.device_scale_factor();
 149
 150   const gfx::ImageSkiaRep& rep =
 151       g_images.Get().mic_empty()->GetRepresentation(scale_);
 152   mic_image_.reset(new SkBitmap());
 153   mic_image_->setConfig(SkBitmap::kARGB_8888_Config,
 154       rep.pixel_width(), rep.pixel_height());
 155   mic_image_->allocPixels();
 156
 157   buffer_image_.reset(new SkBitmap());
 158   buffer_image_->setConfig(SkBitmap::kARGB_8888_Config,
 159       rep.pixel_width(), rep.pixel_height());
 160   buffer_image_->allocPixels();
 161 }
 162
 163 SpeechRecognitionBubbleBase::~SpeechRecognitionBubbleBase() {
 164   // This destructor is added to make sure members such as the scoped_ptr
 165   // get destroyed here and the derived classes don't have to care about such
 166   // member variables which they don't use.
 167 }
 168
 169 void SpeechRecognitionBubbleBase::SetWarmUpMode() {
 170   weak_factory_.InvalidateWeakPtrs();
 171   display_mode_ = DISPLAY_MODE_WARM_UP;
 172   animation_step_ = 0;
 173   DoWarmingUpAnimationStep();
 174   UpdateLayout();
 175 }
 176
 177 void SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep() {
 178   SetImage(g_images.Get().warm_up()[animation_step_]);
 179   base::MessageLoop::current()->PostDelayedTask(
 180       FROM_HERE,
 181       base::Bind(&SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep,
 182           weak_factory_.GetWeakPtr()),
 183       base::TimeDelta::FromMilliseconds(
 184           animation_step_ == 0 ? kWarmingUpAnimationStartMs
 185                                : kWarmingUpAnimationStepMs));
 186   if (++animation_step_ >= static_cast<int>(g_images.Get().warm_up().size()))
 187     animation_step_ = 1;  // Frame 0 is skipped during the animation.
 188 }
 189
 190 void SpeechRecognitionBubbleBase::SetRecordingMode() {
 191   weak_factory_.InvalidateWeakPtrs();
 192   display_mode_ = DISPLAY_MODE_RECORDING;
 193   SetInputVolume(0, 0);
 194   UpdateLayout();
 195 }
 196
 197 void SpeechRecognitionBubbleBase::SetRecognizingMode() {
 198   display_mode_ = DISPLAY_MODE_RECOGNIZING;
 199   animation_step_ = 0;
 200   DoRecognizingAnimationStep();
 201   UpdateLayout();
 202 }
 203
 204 void SpeechRecognitionBubbleBase::DoRecognizingAnimationStep() {
 205   SetImage(g_images.Get().spinner()[animation_step_]);
 206   if (++animation_step_ >= static_cast<int>(g_images.Get().spinner().size()))
 207     animation_step_ = 0;
 208   base::MessageLoop::current()->PostDelayedTask(
 209       FROM_HERE,
 210       base::Bind(&SpeechRecognitionBubbleBase::DoRecognizingAnimationStep,
 211           weak_factory_.GetWeakPtr()),
 212       base::TimeDelta::FromMilliseconds(kRecognizingAnimationStepMs));
 213 }
 214
 215 void SpeechRecognitionBubbleBase::SetMessage(const base::string16& text) {
 216   weak_factory_.InvalidateWeakPtrs();
 217   message_text_ = text;
 218   display_mode_ = DISPLAY_MODE_MESSAGE;
 219   UpdateLayout();
 220 }
 221
 222 void SpeechRecognitionBubbleBase::DrawVolumeOverlay(SkCanvas* canvas,
 223                                                     const gfx::ImageSkia& image,
 224                                                     float volume) {
 225   buffer_image_->eraseARGB(0, 0, 0, 0);
 226
 227   int width = mic_image_->width();
 228   int height = mic_image_->height();
 229   SkCanvas buffer_canvas(*buffer_image_);
 230
 231   buffer_canvas.save();
 232   const int kVolumeSteps = 12;
 233   SkScalar clip_right =
 234       (((1.0f - volume) * (width * (kVolumeSteps + 1))) - width) / kVolumeSteps;
 235   buffer_canvas.clipRect(SkRect::MakeLTRB(0, 0,
 236       SkIntToScalar(width) - clip_right, SkIntToScalar(height)));
 237   buffer_canvas.drawBitmap(image.GetRepresentation(scale_).sk_bitmap(), 0, 0);
 238   buffer_canvas.restore();
 239   SkPaint multiply_paint;
 240   multiply_paint.setXfermodeMode(SkXfermode::kModulate_Mode);
 241   buffer_canvas.drawBitmap(
 242       g_images.Get().mic_mask()->GetRepresentation(scale_).sk_bitmap(),
 243       -clip_right, 0, &multiply_paint);
 244
 245   canvas->drawBitmap(*buffer_image_.get(), 0, 0);
 246 }
 247
 248 void SpeechRecognitionBubbleBase::SetInputVolume(float volume,
 249                                                  float noise_volume) {
 250   mic_image_->eraseARGB(0, 0, 0, 0);
 251   SkCanvas canvas(*mic_image_);
 252
 253   // Draw the empty volume image first and the current volume image on top,
 254   // and then the noise volume image on top of both.
 255   canvas.drawBitmap(
 256       g_images.Get().mic_empty()->GetRepresentation(scale_).sk_bitmap(),
 257       0, 0);
 258   DrawVolumeOverlay(&canvas, *g_images.Get().mic_full(), volume);
 259   DrawVolumeOverlay(&canvas, *g_images.Get().mic_noise(), noise_volume);
 260
 261   gfx::ImageSkia image(gfx::ImageSkiaRep(*mic_image_.get(), scale_));
 262   SetImage(image);
 263 }
 264
 265 WebContents* SpeechRecognitionBubbleBase::GetWebContents() {
 266   return web_contents_;
 267 }
 268
 269 void SpeechRecognitionBubbleBase::SetImage(const gfx::ImageSkia& image) {
 270   icon_image_ = image;
 271   UpdateImage();
 272 }
 273
 274 gfx::ImageSkia SpeechRecognitionBubbleBase::icon_image() {
 275   return icon_image_;
 276 }