media/filters/skcanvas_video_renderer.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/skcanvas_video_renderer.h"
   6
   7 #include "base/logging.h"
   8 #include "media/base/video_frame.h"
   9 #include "media/base/yuv_convert.h"
  10 #include "third_party/libyuv/include/libyuv.h"
  11 #include "third_party/skia/include/core/SkCanvas.h"
  12 #include "third_party/skia/include/core/SkImageGenerator.h"
  13 #include "third_party/skia/include/gpu/GrContext.h"
  14 #include "ui/gfx/skbitmap_operations.h"
  15
  16 // Skia internal format depends on a platform. On Android it is ABGR, on others
  17 // it is ARGB.
  18 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
  19     SK_A32_SHIFT == 24
  20 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
  21 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
  22 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
  23     SK_A32_SHIFT == 24
  24 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
  25 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
  26 #else
  27 #error Unexpected Skia ARGB_8888 layout!
  28 #endif
  29
  30 namespace media {
  31
  32 namespace {
  33
  34 // This class keeps two temporary resources; software bitmap, hardware bitmap.
  35 // If both bitmap are created and then only software bitmap is updated every
  36 // frame, hardware bitmap outlives until the media player dies. So we delete
  37 // a temporary resource if it is not used for 3 sec.
  38 const int kTemporaryResourceDeletionDelay = 3;  // Seconds;
  39
  40 bool IsYUV(media::VideoFrame::Format format) {
  41   switch (format) {
  42     case VideoFrame::YV12:
  43     case VideoFrame::YV16:
  44     case VideoFrame::I420:
  45     case VideoFrame::YV12A:
  46     case VideoFrame::YV12J:
  47     case VideoFrame::YV24:
  48     case VideoFrame::NV12:
  49       return true;
  50     case VideoFrame::UNKNOWN:
  51     case VideoFrame::NATIVE_TEXTURE:
  52 #if defined(VIDEO_HOLE)
  53     case VideoFrame::HOLE:
  54 #endif  // defined(VIDEO_HOLE)
  55       return false;
  56   }
  57   NOTREACHED() << "Invalid videoframe format provided: " << format;
  58   return false;
  59 }
  60
  61 bool IsJPEGColorSpace(media::VideoFrame::Format format) {
  62   switch (format) {
  63     case VideoFrame::YV12J:
  64       return true;
  65     case VideoFrame::YV12:
  66     case VideoFrame::YV16:
  67     case VideoFrame::I420:
  68     case VideoFrame::YV12A:
  69     case VideoFrame::YV24:
  70     case VideoFrame::NV12:
  71     case VideoFrame::UNKNOWN:
  72     case VideoFrame::NATIVE_TEXTURE:
  73 #if defined(VIDEO_HOLE)
  74     case VideoFrame::HOLE:
  75 #endif  // defined(VIDEO_HOLE)
  76       return false;
  77   }
  78   NOTREACHED() << "Invalid videoframe format provided: " << format;
  79   return false;
  80 }
  81
  82 bool IsYUVOrNative(media::VideoFrame::Format format) {
  83   return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
  84 }
  85
  86 // Converts a |video_frame| to raw |rgb_pixels|.
  87 void ConvertVideoFrameToRGBPixels(
  88     const scoped_refptr<media::VideoFrame>& video_frame,
  89     void* rgb_pixels,
  90     size_t row_bytes) {
  91   DCHECK(IsYUVOrNative(video_frame->format()))
  92       << video_frame->format();
  93   if (IsYUV(video_frame->format())) {
  94     DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane),
  95               video_frame->stride(media::VideoFrame::kVPlane));
  96   }
  97
  98   size_t y_offset = 0;
  99   size_t uv_offset = 0;
 100   if (IsYUV(video_frame->format())) {
 101     int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1;
 102     // Use the "left" and "top" of the destination rect to locate the offset
 103     // in Y, U and V planes.
 104     y_offset = (video_frame->stride(media::VideoFrame::kYPlane) *
 105                 video_frame->visible_rect().y()) +
 106                 video_frame->visible_rect().x();
 107     // For format YV12, there is one U, V value per 2x2 block.
 108     // For format YV16, there is one U, V value per 2x1 block.
 109     uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) *
 110                 (video_frame->visible_rect().y() >> y_shift)) +
 111                 (video_frame->visible_rect().x() >> 1);
 112   }
 113
 114   switch (video_frame->format()) {
 115     case media::VideoFrame::YV12:
 116     case media::VideoFrame::I420:
 117       LIBYUV_I420_TO_ARGB(
 118           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 119           video_frame->stride(media::VideoFrame::kYPlane),
 120           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 121           video_frame->stride(media::VideoFrame::kUPlane),
 122           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 123           video_frame->stride(media::VideoFrame::kVPlane),
 124           static_cast<uint8*>(rgb_pixels),
 125           row_bytes,
 126           video_frame->visible_rect().width(),
 127           video_frame->visible_rect().height());
 128       break;
 129
 130     case media::VideoFrame::YV12J:
 131       media::ConvertYUVToRGB32(
 132           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 133           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 134           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 135           static_cast<uint8*>(rgb_pixels),
 136           video_frame->visible_rect().width(),
 137           video_frame->visible_rect().height(),
 138           video_frame->stride(media::VideoFrame::kYPlane),
 139           video_frame->stride(media::VideoFrame::kUPlane),
 140           row_bytes,
 141           media::YV12J);
 142       break;
 143
 144     case media::VideoFrame::YV16:
 145       LIBYUV_I422_TO_ARGB(
 146           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 147           video_frame->stride(media::VideoFrame::kYPlane),
 148           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 149           video_frame->stride(media::VideoFrame::kUPlane),
 150           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 151           video_frame->stride(media::VideoFrame::kVPlane),
 152           static_cast<uint8*>(rgb_pixels),
 153           row_bytes,
 154           video_frame->visible_rect().width(),
 155           video_frame->visible_rect().height());
 156       break;
 157
 158     case media::VideoFrame::YV12A:
 159       // Since libyuv doesn't support YUVA, fallback to media, which is not ARM
 160       // optimized.
 161       // TODO(fbarchard, mtomasz): Use libyuv, then copy the alpha channel.
 162       media::ConvertYUVAToARGB(
 163           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 164           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 165           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 166           video_frame->data(media::VideoFrame::kAPlane),
 167           static_cast<uint8*>(rgb_pixels),
 168           video_frame->visible_rect().width(),
 169           video_frame->visible_rect().height(),
 170           video_frame->stride(media::VideoFrame::kYPlane),
 171           video_frame->stride(media::VideoFrame::kUPlane),
 172           video_frame->stride(media::VideoFrame::kAPlane),
 173           row_bytes,
 174           media::YV12);
 175       break;
 176
 177     case media::VideoFrame::YV24:
 178       libyuv::I444ToARGB(
 179           video_frame->data(media::VideoFrame::kYPlane) + y_offset,
 180           video_frame->stride(media::VideoFrame::kYPlane),
 181           video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
 182           video_frame->stride(media::VideoFrame::kUPlane),
 183           video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
 184           video_frame->stride(media::VideoFrame::kVPlane),
 185           static_cast<uint8*>(rgb_pixels),
 186           row_bytes,
 187           video_frame->visible_rect().width(),
 188           video_frame->visible_rect().height());
 189 #if SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
 190     SK_A32_SHIFT == 24
 191       libyuv::ARGBToABGR(static_cast<uint8*>(rgb_pixels),
 192                          row_bytes,
 193                          static_cast<uint8*>(rgb_pixels),
 194                          row_bytes,
 195                          video_frame->visible_rect().width(),
 196                          video_frame->visible_rect().height());
 197 #endif
 198       break;
 199
 200     case media::VideoFrame::NATIVE_TEXTURE: {
 201       DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE);
 202       SkBitmap tmp;
 203       tmp.installPixels(
 204           SkImageInfo::MakeN32Premul(video_frame->visible_rect().width(),
 205                                      video_frame->visible_rect().height()),
 206           rgb_pixels,
 207           row_bytes);
 208       video_frame->ReadPixelsFromNativeTexture(tmp);
 209       break;
 210     }
 211     default:
 212       NOTREACHED();
 213       break;
 214   }
 215 }
 216
 217 }  // anonymous namespace
 218
 219 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 220 class VideoImageGenerator : public SkImageGenerator {
 221  public:
 222   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame) : frame_(frame) {
 223     DCHECK(frame_.get());
 224   }
 225   ~VideoImageGenerator() override {}
 226
 227   void set_frame(const scoped_refptr<VideoFrame>& frame) { frame_ = frame; }
 228
 229  protected:
 230   bool onGetInfo(SkImageInfo* info) override {
 231     info->fWidth = frame_->visible_rect().width();
 232     info->fHeight = frame_->visible_rect().height();
 233     info->fColorType = kN32_SkColorType;
 234     info->fAlphaType = kPremul_SkAlphaType;
 235     return true;
 236   }
 237
 238   bool onGetPixels(const SkImageInfo& info,
 239                    void* pixels,
 240                    size_t row_bytes,
 241                    SkPMColor ctable[],
 242                    int* ctable_count) override {
 243     if (!frame_.get())
 244       return false;
 245     if (!pixels)
 246       return false;
 247     // If skia couldn't do the YUV conversion on GPU, we will on CPU.
 248     ConvertVideoFrameToRGBPixels(frame_, pixels, row_bytes);
 249     return true;
 250   }
 251
 252   bool onGetYUV8Planes(SkISize sizes[3],
 253                        void* planes[3],
 254                        size_t row_bytes[3],
 255                        SkYUVColorSpace* color_space) override {
 256     if (!frame_.get() || !IsYUV(frame_->format()))
 257       return false;
 258
 259     if (color_space) {
 260       if (IsJPEGColorSpace(frame_->format()))
 261         *color_space = kJPEG_SkYUVColorSpace;
 262       else
 263         *color_space = kRec601_SkYUVColorSpace;
 264     }
 265
 266     for (int plane = VideoFrame::kYPlane; plane <= VideoFrame::kVPlane;
 267          ++plane) {
 268       if (sizes) {
 269         gfx::Size size;
 270         size =
 271             VideoFrame::PlaneSize(frame_->format(),
 272                                   plane,
 273                                   gfx::Size(frame_->visible_rect().width(),
 274                                             frame_->visible_rect().height()));
 275         sizes[plane].set(size.width(), size.height());
 276       }
 277       if (row_bytes && planes) {
 278         size_t offset;
 279         int y_shift = (frame_->format() == media::VideoFrame::YV16) ? 0 : 1;
 280         if (plane == media::VideoFrame::kYPlane) {
 281           offset = (frame_->stride(media::VideoFrame::kYPlane) *
 282                     frame_->visible_rect().y()) +
 283                    frame_->visible_rect().x();
 284         } else {
 285           offset = (frame_->stride(media::VideoFrame::kUPlane) *
 286                     (frame_->visible_rect().y() >> y_shift)) +
 287                    (frame_->visible_rect().x() >> 1);
 288         }
 289         row_bytes[plane] = static_cast<size_t>(frame_->stride(plane));
 290         planes[plane] = frame_->data(plane) + offset;
 291       }
 292     }
 293     return true;
 294   }
 295
 296  private:
 297   scoped_refptr<VideoFrame> frame_;
 298
 299   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 300 };
 301
 302 SkCanvasVideoRenderer::SkCanvasVideoRenderer()
 303     : last_frame_timestamp_(media::kNoTimestamp()),
 304       frame_deleting_timer_(
 305           FROM_HERE,
 306           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
 307           this,
 308           &SkCanvasVideoRenderer::ResetLastFrame),
 309       accelerated_generator_(NULL),
 310       accelerated_last_frame_timestamp_(media::kNoTimestamp()),
 311       accelerated_frame_deleting_timer_(
 312           FROM_HERE,
 313           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
 314           this,
 315           &SkCanvasVideoRenderer::ResetAcceleratedLastFrame) {
 316   last_frame_.setIsVolatile(true);
 317 }
 318
 319 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
 320
 321 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
 322                                   SkCanvas* canvas,
 323                                   const gfx::RectF& dest_rect,
 324                                   uint8 alpha,
 325                                   SkXfermode::Mode mode,
 326                                   VideoRotation video_rotation) {
 327   if (alpha == 0) {
 328     return;
 329   }
 330
 331   SkRect dest;
 332   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 333
 334   SkPaint paint;
 335   paint.setAlpha(alpha);
 336
 337   // Paint black rectangle if there isn't a frame available or the
 338   // frame has an unexpected format.
 339   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
 340       !IsYUVOrNative(video_frame->format())) {
 341     canvas->drawRect(dest, paint);
 342     canvas->flush();
 343     return;
 344   }
 345
 346   SkBitmap* target_frame = NULL;
 347   if (canvas->getGrContext()) {
 348     if (accelerated_last_frame_.isNull() ||
 349         video_frame->timestamp() != accelerated_last_frame_timestamp_) {
 350       accelerated_generator_ = new VideoImageGenerator(video_frame);
 351
 352       // Note: This takes ownership of |accelerated_generator_|.
 353       if (!SkInstallDiscardablePixelRef(accelerated_generator_,
 354                                         &accelerated_last_frame_)) {
 355         NOTREACHED();
 356       }
 357       DCHECK(video_frame->visible_rect().width() ==
 358                  accelerated_last_frame_.width() &&
 359              video_frame->visible_rect().height() ==
 360                  accelerated_last_frame_.height());
 361
 362       accelerated_last_frame_timestamp_ = video_frame->timestamp();
 363     } else {
 364       accelerated_generator_->set_frame(video_frame);
 365     }
 366     target_frame = &accelerated_last_frame_;
 367     accelerated_frame_deleting_timer_.Reset();
 368   } else {
 369     // Check if we should convert and update |last_frame_|.
 370     if (last_frame_.isNull() ||
 371         video_frame->timestamp() != last_frame_timestamp_) {
 372       // Check if |bitmap| needs to be (re)allocated.
 373       if (last_frame_.isNull() ||
 374           last_frame_.width() != video_frame->visible_rect().width() ||
 375           last_frame_.height() != video_frame->visible_rect().height()) {
 376         last_frame_.allocN32Pixels(video_frame->visible_rect().width(),
 377                                    video_frame->visible_rect().height());
 378         last_frame_.setIsVolatile(true);
 379       }
 380       last_frame_.lockPixels();
 381       ConvertVideoFrameToRGBPixels(
 382           video_frame, last_frame_.getPixels(), last_frame_.rowBytes());
 383       last_frame_.notifyPixelsChanged();
 384       last_frame_.unlockPixels();
 385       last_frame_timestamp_ = video_frame->timestamp();
 386     }
 387     target_frame = &last_frame_;
 388     frame_deleting_timer_.Reset();
 389   }
 390
 391   paint.setXfermodeMode(mode);
 392   paint.setFilterLevel(SkPaint::kLow_FilterLevel);
 393
 394   bool need_transform =
 395       video_rotation != VIDEO_ROTATION_0 ||
 396       dest_rect.size() != video_frame->visible_rect().size() ||
 397       !dest_rect.origin().IsOrigin();
 398   if (need_transform) {
 399     canvas->save();
 400     canvas->translate(
 401         SkFloatToScalar(dest_rect.x() + (dest_rect.width() * 0.5f)),
 402         SkFloatToScalar(dest_rect.y() + (dest_rect.height() * 0.5f)));
 403     SkScalar angle = SkFloatToScalar(0.0f);
 404     switch (video_rotation) {
 405       case VIDEO_ROTATION_0:
 406         break;
 407       case VIDEO_ROTATION_90:
 408         angle = SkFloatToScalar(90.0f);
 409         break;
 410       case VIDEO_ROTATION_180:
 411         angle = SkFloatToScalar(180.0f);
 412         break;
 413       case VIDEO_ROTATION_270:
 414         angle = SkFloatToScalar(270.0f);
 415         break;
 416     }
 417     canvas->rotate(angle);
 418
 419     gfx::SizeF rotated_dest_size = dest_rect.size();
 420     if (video_rotation == VIDEO_ROTATION_90 ||
 421         video_rotation == VIDEO_ROTATION_270) {
 422       rotated_dest_size =
 423           gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width());
 424     }
 425     canvas->scale(
 426         SkFloatToScalar(rotated_dest_size.width() / target_frame->width()),
 427         SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
 428     canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
 429                       -SkFloatToScalar(target_frame->height() * 0.5f));
 430   }
 431   canvas->drawBitmap(*target_frame, 0, 0, &paint);
 432   if (need_transform)
 433     canvas->restore();
 434   canvas->flush();
 435   // SkCanvas::flush() causes the generator to generate SkImage, so delete
 436   // |video_frame| not to be outlived.
 437   if (canvas->getGrContext())
 438     accelerated_generator_->set_frame(NULL);
 439 }
 440
 441 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
 442                                  SkCanvas* canvas) {
 443   Paint(video_frame,
 444         canvas,
 445         video_frame->visible_rect(),
 446         0xff,
 447         SkXfermode::kSrc_Mode,
 448         media::VIDEO_ROTATION_0);
 449 }
 450
 451 void SkCanvasVideoRenderer::ResetLastFrame() {
 452   last_frame_.reset();
 453   last_frame_timestamp_ = media::kNoTimestamp();
 454 }
 455
 456 void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
 457   accelerated_last_frame_.reset();
 458   accelerated_generator_ = nullptr;
 459   accelerated_last_frame_timestamp_ = media::kNoTimestamp();
 460 }
 461
 462 }  // namespace media