chrome/browser/speech/speech_recognition_bubble.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "chrome/browser/speech/speech_recognition_bubble.h"
   6
   7 #include "base/bind.h"
   8 #include "base/lazy_instance.h"
   9 #include "base/message_loop.h"
  10 #include "content/public/browser/web_contents.h"
  11 #include "content/public/browser/web_contents_view.h"
  12 #include "grit/generated_resources.h"
  13 #include "grit/theme_resources.h"
  14 #include "ui/base/resource/resource_bundle.h"
  15 #include "ui/gfx/canvas.h"
  16 #include "ui/gfx/display.h"
  17 #include "ui/gfx/image/image_skia_operations.h"
  18 #include "ui/gfx/rect.h"
  19 #include "ui/gfx/screen.h"
  20
  21 using content::WebContents;
  22
  23 namespace {
  24
  25 const color_utils::HSL kGrayscaleShift = { -1, 0, 0.6 };
  26 const int kWarmingUpAnimationStartMs = 500;
  27 const int kWarmingUpAnimationStepMs = 100;
  28 const int kRecognizingAnimationStepMs = 100;
  29
  30 // A lazily initialized singleton to hold all the image used by the speech
  31 // recognition bubbles and safely destroy them on exit.
  32 class SpeechRecognitionBubbleImages {
  33  public:
  34   const std::vector<gfx::ImageSkia>& spinner() const { return spinner_; }
  35   const std::vector<gfx::ImageSkia>& warm_up() const { return warm_up_; }
  36   gfx::ImageSkia* mic_full() const { return mic_full_; }
  37   gfx::ImageSkia* mic_empty() const { return mic_empty_; }
  38   gfx::ImageSkia* mic_noise() const { return mic_noise_; }
  39   gfx::ImageSkia* mic_mask() const { return mic_mask_; }
  40
  41  private:
  42   // Private constructor to enforce singleton.
  43   friend struct base::DefaultLazyInstanceTraits<SpeechRecognitionBubbleImages>;
  44   SpeechRecognitionBubbleImages();
  45
  46   std::vector<gfx::ImageSkia> spinner_;  // Frames for the progress spinner.
  47   std::vector<gfx::ImageSkia> warm_up_;  // Frames for the warm up animation.
  48
  49   // These images are owned by ResourceBundle and need not be destroyed.
  50   gfx::ImageSkia* mic_full_;  // Mic image with full volume.
  51   gfx::ImageSkia* mic_noise_;  // Mic image with full noise volume.
  52   gfx::ImageSkia* mic_empty_;  // Mic image with zero volume.
  53   gfx::ImageSkia* mic_mask_;  // Gradient mask used by the volume indicator.
  54 };
  55
  56 SpeechRecognitionBubbleImages::SpeechRecognitionBubbleImages() {
  57   mic_empty_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  58       IDR_SPEECH_INPUT_MIC_EMPTY);
  59   mic_noise_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  60       IDR_SPEECH_INPUT_MIC_NOISE);
  61   mic_full_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  62       IDR_SPEECH_INPUT_MIC_FULL);
  63   mic_mask_ = ui::ResourceBundle::GetSharedInstance().GetImageSkiaNamed(
  64       IDR_SPEECH_INPUT_MIC_MASK);
  65
  66   // The sprite image consists of all the animation frames put together in one
  67   // horizontal/wide image. Each animation frame is square in shape within the
  68   // sprite.
  69   const gfx::ImageSkia* spinner_image = ui::ResourceBundle::GetSharedInstance().
  70       GetImageSkiaNamed(IDR_SPEECH_INPUT_SPINNER);
  71   int frame_size = spinner_image->height();
  72
  73   // When recording starts up, it may take a short while (few ms or even a
  74   // couple of seconds) before the audio device starts really capturing data.
  75   // This is more apparent on first use. To cover such cases we show a warming
  76   // up state in the bubble starting with a blank spinner image. If audio data
  77   // starts coming in within a couple hundred ms, we switch to the recording
  78   // UI and if it takes longer, we show the real warm up animation frames.
  79   // This reduces visual jank for the most part.
  80   SkBitmap empty_spinner;
  81   empty_spinner.setConfig(SkBitmap::kARGB_8888_Config, frame_size, frame_size);
  82   empty_spinner.allocPixels();
  83   empty_spinner.eraseRGB(255, 255, 255);
  84   // |empty_spinner| has solid color. Pixel doubling a solid color is ok.
  85   warm_up_.push_back(gfx::ImageSkia::CreateFrom1xBitmap(empty_spinner));
  86
  87   for (gfx::Rect src_rect(frame_size, frame_size);
  88        src_rect.x() < spinner_image->width();
  89        src_rect.Offset(frame_size, 0)) {
  90     gfx::ImageSkia frame = gfx::ImageSkiaOperations::ExtractSubset(
  91         *spinner_image, src_rect);
  92
  93     // The image created by ExtractSubset just points to the same pixels as
  94     // the original and adjusts rowBytes accordingly. However that doesn't
  95     // render properly and gets vertically squished in Linux due to a bug in
  96     // Skia. Until that gets fixed we work around by taking a real copy of it
  97     // below as the copied image has the correct rowBytes and renders fine.
  98     frame.EnsureRepsForSupportedScaleFactors();
  99     std::vector<gfx::ImageSkiaRep> image_reps = frame.image_reps();
 100     gfx::ImageSkia frame_copy;
 101     for (size_t i = 0; i < image_reps.size(); ++i) {
 102       const SkBitmap& copy_src = image_reps[i].sk_bitmap();
 103       SkBitmap copy_dst;
 104       copy_src.copyTo(&copy_dst, SkBitmap::kARGB_8888_Config);
 105       frame_copy.AddRepresentation(gfx::ImageSkiaRep(
 106           copy_dst, image_reps[i].scale_factor()));
 107     }
 108     spinner_.push_back(frame_copy);
 109
 110     // The warm up spinner animation is a gray scale version of the real one.
 111     warm_up_.push_back(gfx::ImageSkiaOperations::CreateHSLShiftedImage(
 112         frame_copy, kGrayscaleShift));
 113   }
 114 }
 115
 116 base::LazyInstance<SpeechRecognitionBubbleImages> g_images =
 117     LAZY_INSTANCE_INITIALIZER;
 118
 119 }  // namespace
 120
 121 SpeechRecognitionBubble::FactoryMethod SpeechRecognitionBubble::factory_ = NULL;
 122 const int SpeechRecognitionBubble::kBubbleTargetOffsetX = 10;
 123
 124 SpeechRecognitionBubble* SpeechRecognitionBubble::Create(
 125     WebContents* web_contents, Delegate* delegate,
 126     const gfx::Rect& element_rect) {
 127   if (factory_)
 128     return (*factory_)(web_contents, delegate, element_rect);
 129
 130   // Has the tab already closed before bubble create request was processed?
 131   if (!web_contents)
 132     return NULL;
 133
 134   return CreateNativeBubble(web_contents, delegate, element_rect);
 135 }
 136
 137 SpeechRecognitionBubbleBase::SpeechRecognitionBubbleBase(
 138     WebContents* web_contents)
 139     : ALLOW_THIS_IN_INITIALIZER_LIST(weak_factory_(this)),
 140       animation_step_(0),
 141       display_mode_(DISPLAY_MODE_RECORDING),
 142       web_contents_(web_contents),
 143       scale_factor_(ui::SCALE_FACTOR_NONE) {
 144   gfx::NativeView view =
 145       web_contents_ ? web_contents_->GetView()->GetNativeView() : NULL;
 146   gfx::Screen* screen = gfx::Screen::GetScreenFor(view);
 147   gfx::Display display = screen->GetDisplayNearestWindow(view);
 148   scale_factor_ = ui::GetScaleFactorFromScale(
 149       display.device_scale_factor());
 150
 151   const gfx::ImageSkiaRep& rep =
 152       g_images.Get().mic_empty()->GetRepresentation(scale_factor_);
 153   mic_image_.reset(new SkBitmap());
 154   mic_image_->setConfig(SkBitmap::kARGB_8888_Config,
 155       rep.pixel_width(), rep.pixel_height());
 156   mic_image_->allocPixels();
 157
 158   buffer_image_.reset(new SkBitmap());
 159   buffer_image_->setConfig(SkBitmap::kARGB_8888_Config,
 160       rep.pixel_width(), rep.pixel_height());
 161   buffer_image_->allocPixels();
 162 }
 163
 164 SpeechRecognitionBubbleBase::~SpeechRecognitionBubbleBase() {
 165   // This destructor is added to make sure members such as the scoped_ptr
 166   // get destroyed here and the derived classes don't have to care about such
 167   // member variables which they don't use.
 168 }
 169
 170 void SpeechRecognitionBubbleBase::SetWarmUpMode() {
 171   weak_factory_.InvalidateWeakPtrs();
 172   display_mode_ = DISPLAY_MODE_WARM_UP;
 173   animation_step_ = 0;
 174   DoWarmingUpAnimationStep();
 175   UpdateLayout();
 176 }
 177
 178 void SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep() {
 179   SetImage(g_images.Get().warm_up()[animation_step_]);
 180   MessageLoop::current()->PostDelayedTask(
 181       FROM_HERE,
 182       base::Bind(&SpeechRecognitionBubbleBase::DoWarmingUpAnimationStep,
 183           weak_factory_.GetWeakPtr()),
 184       base::TimeDelta::FromMilliseconds(
 185           animation_step_ == 0 ? kWarmingUpAnimationStartMs
 186                                : kWarmingUpAnimationStepMs));
 187   if (++animation_step_ >= static_cast<int>(g_images.Get().warm_up().size()))
 188     animation_step_ = 1;  // Frame 0 is skipped during the animation.
 189 }
 190
 191 void SpeechRecognitionBubbleBase::SetRecordingMode() {
 192   weak_factory_.InvalidateWeakPtrs();
 193   display_mode_ = DISPLAY_MODE_RECORDING;
 194   SetInputVolume(0, 0);
 195   UpdateLayout();
 196 }
 197
 198 void SpeechRecognitionBubbleBase::SetRecognizingMode() {
 199   display_mode_ = DISPLAY_MODE_RECOGNIZING;
 200   animation_step_ = 0;
 201   DoRecognizingAnimationStep();
 202   UpdateLayout();
 203 }
 204
 205 void SpeechRecognitionBubbleBase::DoRecognizingAnimationStep() {
 206   SetImage(g_images.Get().spinner()[animation_step_]);
 207   if (++animation_step_ >= static_cast<int>(g_images.Get().spinner().size()))
 208     animation_step_ = 0;
 209   MessageLoop::current()->PostDelayedTask(
 210       FROM_HERE,
 211       base::Bind(&SpeechRecognitionBubbleBase::DoRecognizingAnimationStep,
 212           weak_factory_.GetWeakPtr()),
 213       base::TimeDelta::FromMilliseconds(kRecognizingAnimationStepMs));
 214 }
 215
 216 void SpeechRecognitionBubbleBase::SetMessage(const string16& text) {
 217   weak_factory_.InvalidateWeakPtrs();
 218   message_text_ = text;
 219   display_mode_ = DISPLAY_MODE_MESSAGE;
 220   UpdateLayout();
 221 }
 222
 223 void SpeechRecognitionBubbleBase::DrawVolumeOverlay(SkCanvas* canvas,
 224                                                     const gfx::ImageSkia& image,
 225                                                     float volume) {
 226   buffer_image_->eraseARGB(0, 0, 0, 0);
 227
 228   int width = mic_image_->width();
 229   int height = mic_image_->height();
 230   SkCanvas buffer_canvas(*buffer_image_);
 231
 232   buffer_canvas.save();
 233   const int kVolumeSteps = 12;
 234   SkScalar clip_right =
 235       (((1.0f - volume) * (width * (kVolumeSteps + 1))) - width) / kVolumeSteps;
 236   buffer_canvas.clipRect(SkRect::MakeLTRB(0, 0,
 237       SkIntToScalar(width) - clip_right, SkIntToScalar(height)));
 238   buffer_canvas.drawBitmap(
 239       image.GetRepresentation(scale_factor_).sk_bitmap(), 0, 0);
 240   buffer_canvas.restore();
 241   SkPaint multiply_paint;
 242   multiply_paint.setXfermode(SkXfermode::Create(SkXfermode::kModulate_Mode));
 243   buffer_canvas.drawBitmap(
 244       g_images.Get().mic_mask()->GetRepresentation(scale_factor_).sk_bitmap(),
 245       -clip_right, 0, &multiply_paint);
 246
 247   canvas->drawBitmap(*buffer_image_.get(), 0, 0);
 248 }
 249
 250 void SpeechRecognitionBubbleBase::SetInputVolume(float volume,
 251                                                  float noise_volume) {
 252   mic_image_->eraseARGB(0, 0, 0, 0);
 253   SkCanvas canvas(*mic_image_);
 254
 255   // Draw the empty volume image first and the current volume image on top,
 256   // and then the noise volume image on top of both.
 257   canvas.drawBitmap(
 258       g_images.Get().mic_empty()->GetRepresentation(scale_factor_).sk_bitmap(),
 259       0, 0);
 260   DrawVolumeOverlay(&canvas, *g_images.Get().mic_full(), volume);
 261   DrawVolumeOverlay(&canvas, *g_images.Get().mic_noise(), noise_volume);
 262
 263   gfx::ImageSkia image(gfx::ImageSkiaRep(*mic_image_.get(), scale_factor_));
 264   SetImage(image);
 265 }
 266
 267 WebContents* SpeechRecognitionBubbleBase::GetWebContents() {
 268   return web_contents_;
 269 }
 270
 271 void SpeechRecognitionBubbleBase::SetImage(const gfx::ImageSkia& image) {
 272   icon_image_ = image;
 273   UpdateImage();
 274 }
 275
 276 gfx::ImageSkia SpeechRecognitionBubbleBase::icon_image() {
 277   return icon_image_;
 278 }