media/filters/skcanvas_video_renderer.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/skcanvas_video_renderer.h"
   6
   7 #include "base/logging.h"
   8 #include "media/base/video_frame.h"
   9 #include "media/base/yuv_convert.h"
  10 #include "third_party/libyuv/include/libyuv.h"
  11 #include "third_party/skia/include/core/SkCanvas.h"
  12 #include "third_party/skia/include/core/SkImageGenerator.h"
  13 #include "ui/gfx/skbitmap_operations.h"
  14
  15 // Skia internal format depends on a platform. On Android it is ABGR, on others
  16 // it is ARGB.
  17 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
  18     SK_A32_SHIFT == 24
  19 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
  20 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
  21 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
  22     SK_A32_SHIFT == 24
  23 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
  24 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
  25 #else
  26 #error Unexpected Skia ARGB_8888 layout!
  27 #endif
  28
  29 namespace media {
  30
  31 static bool IsYUV(media::VideoFrame::Format format) {
  32   switch (format) {
  33     case VideoFrame::YV12:
  34     case VideoFrame::YV16:
  35     case VideoFrame::I420:
  36     case VideoFrame::YV12A:
  37     case VideoFrame::YV12J:
  38     case VideoFrame::YV24:
  39     case VideoFrame::NV12:
  40       return true;
  41     case VideoFrame::UNKNOWN:
  42     case VideoFrame::NATIVE_TEXTURE:
  43 #if defined(VIDEO_HOLE)
  44     case VideoFrame::HOLE:
  45 #endif  // defined(VIDEO_HOLE)
  46       return false;
  47   }
  48   NOTREACHED() << "Invalid videoframe format provided: " << format;
  49   return false;
  50 }
  51
  52 static bool IsJPEGColorSpace(media::VideoFrame::Format format) {
  53   switch (format) {
  54     case VideoFrame::YV12J:
  55       return true;
  56     case VideoFrame::YV12:
  57     case VideoFrame::YV16:
  58     case VideoFrame::I420:
  59     case VideoFrame::YV12A:
  60     case VideoFrame::YV24:
  61     case VideoFrame::NV12:
  62     case VideoFrame::UNKNOWN:
  63     case VideoFrame::NATIVE_TEXTURE:
  64 #if defined(VIDEO_HOLE)
  65     case VideoFrame::HOLE:
  66 #endif  // defined(VIDEO_HOLE)
  67       return false;
  68   }
  69   NOTREACHED() << "Invalid videoframe format provided: " << format;
  70   return false;
  71 }
  72
  73 static bool IsYUVOrNative(media::VideoFrame::Format format) {
  74   return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
  75 }
  76
  77 // Converts a |video_frame| to raw |rgb_pixels|.
  78 static void ConvertVideoFrameToRGBPixels(
  79     const scoped_refptr<media::VideoFrame>& video_frame,
  80     void* rgb_pixels,
  81     size_t row_bytes) {
  82   DCHECK(IsYUVOrNative(video_frame->format()))
  83       << video_frame->format();
  84   if (IsYUV(video_frame->format())) {
  85     DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane),
  86               video_frame->stride(media::VideoFrame::kVPlane));
  87   }
  88
  89   size_t y_offset = 0;
  90   size_t uv_offset = 0;
  91   if (IsYUV(video_frame->format())) {
  92     int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1;
  93     // Use the "left" and "top" of the destination rect to locate the offset
  94     // in Y, U and V planes.
  95     y_offset = (video_frame->stride(media::VideoFrame::kYPlane) *
  96                 video_frame->visible_rect().y()) +
  97                 video_frame->visible_rect().x();
  98     // For format YV12, there is one U, V value per 2x2 block.
  99     // For format YV16, there is one U, V value per 2x1 block.
 100     uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) *
 101                 (video_frame->visible_rect().y() >> y_shift)) +
 102                 (video_frame->visible_rect().x() >> 1);
 103   }
 104
 105   switch (video_frame->format()) {
 106     case media::VideoFrame::YV12:
 107     case media::VideoFrame::I420:
 108       LIBYUV_I420_TO_ARGB(
 109           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 110           video_frame->stride(media::VideoFrame::kYPlane),
 111           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 112           video_frame->stride(media::VideoFrame::kUPlane),
 113           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 114           video_frame->stride(media::VideoFrame::kVPlane),
 115           static_cast<uint8*>(rgb_pixels),
 116           row_bytes,
 117           video_frame->visible_rect().width(),
 118           video_frame->visible_rect().height());
 119       break;
 120
 121     case media::VideoFrame::YV12J:
 122       media::ConvertYUVToRGB32(
 123           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 124           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 125           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 126           static_cast<uint8*>(rgb_pixels),
 127           video_frame->visible_rect().width(),
 128           video_frame->visible_rect().height(),
 129           video_frame->stride(media::VideoFrame::kYPlane),
 130           video_frame->stride(media::VideoFrame::kUPlane),
 131           row_bytes,
 132           media::YV12J);
 133       break;
 134
 135     case media::VideoFrame::YV16:
 136       LIBYUV_I422_TO_ARGB(
 137           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 138           video_frame->stride(media::VideoFrame::kYPlane),
 139           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 140           video_frame->stride(media::VideoFrame::kUPlane),
 141           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 142           video_frame->stride(media::VideoFrame::kVPlane),
 143           static_cast<uint8*>(rgb_pixels),
 144           row_bytes,
 145           video_frame->visible_rect().width(),
 146           video_frame->visible_rect().height());
 147       break;
 148
 149     case media::VideoFrame::YV12A:
 150       // Since libyuv doesn't support YUVA, fallback to media, which is not ARM
 151       // optimized.
 152       // TODO(fbarchard, mtomasz): Use libyuv, then copy the alpha channel.
 153       media::ConvertYUVAToARGB(
 154           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 155           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 156           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 157           video_frame->data(media::VideoFrame::kAPlane),
 158           static_cast<uint8*>(rgb_pixels),
 159           video_frame->visible_rect().width(),
 160           video_frame->visible_rect().height(),
 161           video_frame->stride(media::VideoFrame::kYPlane),
 162           video_frame->stride(media::VideoFrame::kUPlane),
 163           video_frame->stride(media::VideoFrame::kAPlane),
 164           row_bytes,
 165           media::YV12);
 166       break;
 167
 168     case media::VideoFrame::YV24:
 169       libyuv::I444ToARGB(
 170           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 171           video_frame->stride(media::VideoFrame::kYPlane),
 172           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 173           video_frame->stride(media::VideoFrame::kUPlane),
 174           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 175           video_frame->stride(media::VideoFrame::kVPlane),
 176           static_cast<uint8*>(rgb_pixels),
 177           row_bytes,
 178           video_frame->visible_rect().width(),
 179           video_frame->visible_rect().height());
 180 #if SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
 181     SK_A32_SHIFT == 24
 182       libyuv::ARGBToABGR(static_cast<uint8*>(rgb_pixels),
 183                          row_bytes,
 184                          static_cast<uint8*>(rgb_pixels),
 185                          row_bytes,
 186                          video_frame->visible_rect().width(),
 187                          video_frame->visible_rect().height());
 188 #endif
 189       break;
 190
 191     case media::VideoFrame::NATIVE_TEXTURE: {
 192       DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE);
 193       SkBitmap tmp;
 194       tmp.installPixels(
 195           SkImageInfo::MakeN32Premul(video_frame->visible_rect().width(),
 196                                      video_frame->visible_rect().height()),
 197           rgb_pixels,
 198           row_bytes);
 199       video_frame->ReadPixelsFromNativeTexture(tmp);
 200       break;
 201     }
 202     default:
 203       NOTREACHED();
 204       break;
 205   }
 206 }
 207
 208 // Generates an RGB image from a VideoFrame.
 209 class VideoImageGenerator : public SkImageGenerator {
 210  public:
 211   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame) : frame_(frame) {}
 212   virtual ~VideoImageGenerator() {}
 213
 214   void set_frame(const scoped_refptr<VideoFrame>& frame) { frame_ = frame; }
 215
 216  protected:
 217   virtual bool onGetInfo(SkImageInfo* info) OVERRIDE {
 218     info->fWidth = frame_->visible_rect().width();
 219     info->fHeight = frame_->visible_rect().height();
 220     info->fColorType = kN32_SkColorType;
 221     info->fAlphaType = kPremul_SkAlphaType;
 222     return true;
 223   }
 224
 225   virtual bool onGetPixels(const SkImageInfo& info,
 226                            void* pixels,
 227                            size_t row_bytes,
 228                            SkPMColor ctable[],
 229                            int* ctable_count) OVERRIDE {
 230     if (!frame_.get())
 231       return false;
 232     if (!pixels)
 233       return true;
 234     // If skia couldn't do the YUV conversion, we will.
 235     ConvertVideoFrameToRGBPixels(frame_, pixels, row_bytes);
 236     frame_ = NULL;
 237     return true;
 238   }
 239
 240   virtual bool onGetYUV8Planes(SkISize sizes[3],
 241                                void* planes[3],
 242                                size_t row_bytes[3],
 243                                SkYUVColorSpace* color_space) OVERRIDE {
 244     if (!frame_.get() || !IsYUV(frame_->format()))
 245       return false;
 246
 247     if (color_space) {
 248       if (IsJPEGColorSpace(frame_->format()))
 249         *color_space = kJPEG_SkYUVColorSpace;
 250       else
 251         *color_space = kRec601_SkYUVColorSpace;
 252     }
 253
 254     for (int plane = VideoFrame::kYPlane; plane <= VideoFrame::kVPlane;
 255          ++plane) {
 256       if (sizes) {
 257         gfx::Size size;
 258         size =
 259             VideoFrame::PlaneSize(frame_->format(),
 260                                   plane,
 261                                   gfx::Size(frame_->visible_rect().width(),
 262                                             frame_->visible_rect().height()));
 263         sizes[plane].set(size.width(), size.height());
 264       }
 265       if (row_bytes && planes) {
 266         size_t offset;
 267         int y_shift = (frame_->format() == media::VideoFrame::YV16) ? 0 : 1;
 268         if (plane == media::VideoFrame::kYPlane) {
 269           offset = (frame_->stride(media::VideoFrame::kYPlane) *
 270                     frame_->visible_rect().y()) +
 271                    frame_->visible_rect().x();
 272         } else {
 273           offset = (frame_->stride(media::VideoFrame::kUPlane) *
 274                     (frame_->visible_rect().y() >> y_shift)) +
 275                    (frame_->visible_rect().x() >> 1);
 276         }
 277         row_bytes[plane] = static_cast<size_t>(frame_->stride(plane));
 278         planes[plane] = frame_->data(plane) + offset;
 279       }
 280     }
 281     if (planes && row_bytes)
 282       frame_ = NULL;
 283     return true;
 284   }
 285
 286  private:
 287   scoped_refptr<VideoFrame> frame_;
 288 };
 289
 290 SkCanvasVideoRenderer::SkCanvasVideoRenderer()
 291     : generator_(NULL), last_frame_timestamp_(media::kNoTimestamp()) {
 292   last_frame_.setIsVolatile(true);
 293 }
 294
 295 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
 296
 297 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
 298                                   SkCanvas* canvas,
 299                                   const gfx::RectF& dest_rect,
 300                                   uint8 alpha,
 301                                   SkXfermode::Mode mode,
 302                                   VideoRotation video_rotation) {
 303   if (alpha == 0) {
 304     return;
 305   }
 306
 307   SkRect dest;
 308   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 309
 310   SkPaint paint;
 311   paint.setAlpha(alpha);
 312
 313   // Paint black rectangle if there isn't a frame available or the
 314   // frame has an unexpected format.
 315   if (!video_frame.get() || !IsYUVOrNative(video_frame->format())) {
 316     canvas->drawRect(dest, paint);
 317     canvas->flush();
 318     return;
 319   }
 320
 321   // Check if we should convert and update |last_frame_|.
 322   if (last_frame_.isNull() ||
 323       video_frame->timestamp() != last_frame_timestamp_) {
 324     generator_ = new VideoImageGenerator(video_frame);
 325
 326     // Note: This takes ownership of |generator_|.
 327     if (!SkInstallDiscardablePixelRef(generator_, &last_frame_)) {
 328       NOTREACHED();
 329     }
 330
 331     // TODO(rileya): Perform this rotation on the canvas, rather than allocating
 332     // a new bitmap and copying.
 333     switch (video_rotation) {
 334       case VIDEO_ROTATION_0:
 335         break;
 336       case VIDEO_ROTATION_90:
 337         last_frame_ = SkBitmapOperations::Rotate(
 338             last_frame_, SkBitmapOperations::ROTATION_90_CW);
 339         break;
 340       case VIDEO_ROTATION_180:
 341         last_frame_ = SkBitmapOperations::Rotate(
 342             last_frame_, SkBitmapOperations::ROTATION_180_CW);
 343         break;
 344       case VIDEO_ROTATION_270:
 345         last_frame_ = SkBitmapOperations::Rotate(
 346             last_frame_, SkBitmapOperations::ROTATION_270_CW);
 347         break;
 348     }
 349
 350     // We copied the frame into a new bitmap and threw out the old one, so we
 351     // no longer have a |generator_| around. This should be removed when the
 352     // above TODO is addressed.
 353     if (video_rotation != VIDEO_ROTATION_0)
 354       generator_ = NULL;
 355
 356     last_frame_timestamp_ = video_frame->timestamp();
 357   } else if (generator_) {
 358     generator_->set_frame(video_frame);
 359   }
 360
 361   paint.setXfermodeMode(mode);
 362
 363   // Paint using |last_frame_|.
 364   paint.setFilterLevel(SkPaint::kLow_FilterLevel);
 365   canvas->drawBitmapRect(last_frame_, NULL, dest, &paint);
 366   canvas->flush();
 367 }
 368
 369 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
 370                                  SkCanvas* canvas) {
 371   Paint(video_frame,
 372         canvas,
 373         video_frame->visible_rect(),
 374         0xff,
 375         SkXfermode::kSrc_Mode,
 376         media::VIDEO_ROTATION_0);
 377 }
 378
 379 }  // namespace media