media/cast/video_sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/base/video_frame.h"
  13 #include "media/base/video_util.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/transport/cast_transport_config.h"
  16 #include "media/video/video_encode_accelerator.h"
  17
  18 namespace {
  19   // We allocate more input buffers than what is asked for by
  20   // RequireBitstreamBuffers() due to potential threading timing.
  21   static const int kInputBufferExtraCount = 1;
  22   static const int kOutputBufferCount = 3;
  23
  24   void LogFrameEncodedEvent(
  25       const base::TimeTicks& now,
  26       media::cast::CastEnvironment* const cast_environment,
  27       const base::TimeTicks& capture_time) {
  28     cast_environment->Logging()->InsertFrameEvent(
  29         now,
  30         media::cast::kVideoFrameEncoded,
  31         media::cast::GetVideoRtpTimestamp(capture_time),
  32         media::cast::kFrameIdUnknown);
  33   }
  34 }  // namespace
  35
  36 namespace media {
  37 namespace cast {
  38
  39 // Container for the associated data of a video frame being processed.
  40 struct EncodedFrameReturnData {
  41   EncodedFrameReturnData(base::TimeTicks c_time,
  42                          VideoEncoder::FrameEncodedCallback callback) {
  43     capture_time = c_time;
  44     frame_encoded_callback = callback;
  45   }
  46   base::TimeTicks capture_time;
  47   VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  48 };
  49
  50 // The ExternalVideoEncoder class can be deleted directly by cast, while
  51 // LocalVideoEncodeAcceleratorClient stays around long enough to properly shut
  52 // down the VideoEncodeAccelerator.
  53 class LocalVideoEncodeAcceleratorClient
  54     : public VideoEncodeAccelerator::Client,
  55       public base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient> {
  56  public:
  57   LocalVideoEncodeAcceleratorClient(
  58       scoped_refptr<CastEnvironment> cast_environment,
  59       scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories,
  60       const base::WeakPtr<ExternalVideoEncoder>& weak_owner)
  61       : cast_environment_(cast_environment),
  62         gpu_factories_(gpu_factories),
  63         encoder_task_runner_(gpu_factories->GetTaskRunner()),
  64         weak_owner_(weak_owner),
  65         last_encoded_frame_id_(kStartFrameId) {
  66     DCHECK(encoder_task_runner_);
  67   }
  68
  69   // Initialize the real HW encoder.
  70   void Initialize(const VideoSenderConfig& video_config) {
  71     DCHECK(gpu_factories_);
  72     DCHECK(encoder_task_runner_);
  73
  74     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
  75
  76     video_encode_accelerator_ =
  77         gpu_factories_->CreateVideoEncodeAccelerator(this).Pass();
  78     if (!video_encode_accelerator_) return;
  79
  80     VideoCodecProfile output_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN;
  81     switch (video_config.codec) {
  82       case transport::kVp8:
  83         output_profile = media::VP8PROFILE_MAIN;
  84         break;
  85       case transport::kH264:
  86         output_profile = media::H264PROFILE_MAIN;
  87         break;
  88     }
  89     codec_ = video_config.codec;
  90     max_frame_rate_ = video_config.max_frame_rate;
  91
  92     // Asynchronous initialization call; NotifyInitializeDone or NotifyError
  93     // will be called once the HW is initialized.
  94     video_encode_accelerator_->Initialize(
  95         media::VideoFrame::I420,
  96         gfx::Size(video_config.width, video_config.height),
  97         output_profile,
  98         video_config.start_bitrate);
  99   }
 100
 101   // Free the HW.
 102   void Destroy() {
 103     DCHECK(encoder_task_runner_);
 104     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 105
 106     if (video_encode_accelerator_) {
 107       video_encode_accelerator_.release()->Destroy();
 108     }
 109   }
 110
 111   void SetBitRate(uint32 bit_rate) {
 112     DCHECK(encoder_task_runner_);
 113     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 114
 115     video_encode_accelerator_->RequestEncodingParametersChange(
 116         bit_rate, max_frame_rate_);
 117   }
 118
 119   void EncodeVideoFrame(
 120       const scoped_refptr<media::VideoFrame>& video_frame,
 121       const base::TimeTicks& capture_time,
 122       bool key_frame_requested,
 123       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 124     DCHECK(encoder_task_runner_);
 125     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 126
 127     if (input_buffers_free_.empty()) {
 128       NOTREACHED();
 129       VLOG(2) << "EncodeVideoFrame(): drop frame due to no hw buffers";
 130       return;
 131     }
 132     const int index = input_buffers_free_.back();
 133     base::SharedMemory* input_buffer = input_buffers_[index];
 134
 135     // TODO(pwestin): this allocation and copy can be removed once we don't
 136     // pass the video frame through liblingle.
 137
 138     scoped_refptr<media::VideoFrame> frame =
 139         media::VideoFrame::WrapExternalPackedMemory(
 140             video_frame->format(),
 141             video_frame->coded_size(),
 142             video_frame->visible_rect(),
 143             video_frame->natural_size(),
 144             reinterpret_cast<uint8*>(input_buffer->memory()),
 145             input_buffer->mapped_size(),
 146             input_buffer->handle(),
 147             video_frame->GetTimestamp(),
 148             base::Bind(&LocalVideoEncodeAcceleratorClient::FinishedWithInBuffer,
 149             this,
 150             index));
 151
 152     if (!frame) {
 153       VLOG(1) << "EncodeVideoFrame(): failed to create frame";
 154       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 155       return;
 156     }
 157     // Do a stride copy of the input frame to match the input requirements.
 158     media::CopyYPlane(video_frame->data(VideoFrame::kYPlane),
 159                       video_frame->stride(VideoFrame::kYPlane),
 160                       video_frame->natural_size().height(),
 161                       frame.get());
 162     media::CopyUPlane(video_frame->data(VideoFrame::kUPlane),
 163                       video_frame->stride(VideoFrame::kUPlane),
 164                       video_frame->natural_size().height(),
 165                       frame.get());
 166     media::CopyVPlane(video_frame->data(VideoFrame::kVPlane),
 167                       video_frame->stride(VideoFrame::kVPlane),
 168                       video_frame->natural_size().height(),
 169                       frame.get());
 170
 171     encoded_frame_data_storage_.push_back(
 172         EncodedFrameReturnData(capture_time, frame_encoded_callback));
 173
 174     // BitstreamBufferReady will be called once the encoder is done.
 175     video_encode_accelerator_->Encode(frame, key_frame_requested);
 176   }
 177
 178  protected:
 179   virtual void NotifyInitializeDone() OVERRIDE {
 180     DCHECK(encoder_task_runner_);
 181     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 182
 183     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 184         base::Bind(&ExternalVideoEncoder::EncoderInitialized,
 185         weak_owner_));
 186   }
 187
 188   virtual void NotifyError(VideoEncodeAccelerator::Error error) OVERRIDE {
 189     DCHECK(encoder_task_runner_);
 190     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 191     VLOG(1) << "ExternalVideoEncoder NotifyError: " << error;
 192
 193     if (video_encode_accelerator_) {
 194       video_encode_accelerator_.release()->Destroy();
 195     }
 196     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 197         base::Bind(&ExternalVideoEncoder::EncoderError,
 198         weak_owner_));
 199   }
 200
 201   // Called to allocate the input and output buffers.
 202   virtual void RequireBitstreamBuffers(unsigned int input_count,
 203                                        const gfx::Size& input_coded_size,
 204                                        size_t output_buffer_size) OVERRIDE {
 205     DCHECK(encoder_task_runner_);
 206     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 207     DCHECK(video_encode_accelerator_);
 208
 209     for (unsigned int i = 0; i < input_count + kInputBufferExtraCount; ++i) {
 210       base::SharedMemory* shm = gpu_factories_->CreateSharedMemory(
 211           media::VideoFrame::AllocationSize(media::VideoFrame::I420,
 212           input_coded_size));
 213       if (!shm) {
 214         VLOG(1) << "RequireBitstreamBuffers(): failed to create input buffer ";
 215         return;
 216       }
 217       input_buffers_.push_back(shm);
 218       input_buffers_free_.push_back(i);
 219     }
 220
 221     for (int j = 0; j < kOutputBufferCount; ++j) {
 222       base::SharedMemory* shm =
 223           gpu_factories_->CreateSharedMemory(output_buffer_size);
 224       if (!shm) {
 225         VLOG(1) << "RequireBitstreamBuffers(): failed to create input buffer ";
 226         return;
 227       }
 228       output_buffers_.push_back(shm);
 229     }
 230     // Immediately provide all output buffers to the VEA.
 231     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 232       video_encode_accelerator_->UseOutputBitstreamBuffer(
 233           media::BitstreamBuffer(static_cast<int32>(i),
 234           output_buffers_[i]->handle(),
 235           output_buffers_[i]->mapped_size()));
 236     }
 237   }
 238
 239   // Encoder has encoded a frame and it's available in one of out output
 240   // buffers.
 241   virtual void BitstreamBufferReady(int32 bitstream_buffer_id,
 242                                     size_t payload_size,
 243                                     bool key_frame) OVERRIDE {
 244     DCHECK(encoder_task_runner_);
 245     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 246     if (bitstream_buffer_id < 0 ||
 247         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 248       NOTREACHED();
 249       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 250                   << bitstream_buffer_id;
 251       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 252       return;
 253     }
 254     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 255     if (payload_size > output_buffer->mapped_size()) {
 256       NOTREACHED();
 257       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 258               << payload_size;
 259       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 260       return;
 261     }
 262     if (encoded_frame_data_storage_.empty()) {
 263       NOTREACHED();
 264       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 265       return;
 266     }
 267     scoped_ptr<transport::EncodedVideoFrame>
 268         encoded_frame(new transport::EncodedVideoFrame());
 269
 270     encoded_frame->codec = codec_;
 271     encoded_frame->key_frame = key_frame;
 272     encoded_frame->last_referenced_frame_id = last_encoded_frame_id_;
 273     last_encoded_frame_id_++;
 274     encoded_frame->frame_id = last_encoded_frame_id_;
 275     if (key_frame) {
 276       // Self referenced.
 277       encoded_frame->last_referenced_frame_id = encoded_frame->frame_id;
 278     }
 279
 280     encoded_frame->data.insert(
 281         0, static_cast<const char*>(output_buffer->memory()), payload_size);
 282
 283     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 284         base::Bind(encoded_frame_data_storage_.front().frame_encoded_callback,
 285         base::Passed(&encoded_frame),
 286         encoded_frame_data_storage_.front().capture_time));
 287
 288     base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 289     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 290         base::Bind(LogFrameEncodedEvent, now,  cast_environment_,
 291         encoded_frame_data_storage_.front().capture_time));
 292
 293     encoded_frame_data_storage_.pop_front();
 294
 295     // We need to re-add the output buffer to the encoder after we are done
 296     // with it.
 297     video_encode_accelerator_->UseOutputBitstreamBuffer(
 298           media::BitstreamBuffer(bitstream_buffer_id,
 299           output_buffers_[bitstream_buffer_id]->handle(),
 300           output_buffers_[bitstream_buffer_id]->mapped_size()));
 301   }
 302
 303  private:
 304   // Encoder is done with the provided input buffer.
 305   void FinishedWithInBuffer(int input_index) {
 306     DCHECK(encoder_task_runner_);
 307     DCHECK(encoder_task_runner_->RunsTasksOnCurrentThread());
 308     DCHECK_GE(input_index, 0);
 309     DCHECK_LT(input_index, static_cast<int>(input_buffers_.size()));
 310     VLOG(2) << "EncodeFrameFinished(): index=" << input_index;
 311     input_buffers_free_.push_back(input_index);
 312   }
 313
 314   friend class base::RefCountedThreadSafe<LocalVideoEncodeAcceleratorClient>;
 315
 316   virtual ~LocalVideoEncodeAcceleratorClient() {}
 317
 318   const scoped_refptr<CastEnvironment> cast_environment_;
 319   scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories_;
 320   scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
 321   const base::WeakPtr<ExternalVideoEncoder> weak_owner_;
 322
 323   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 324   int max_frame_rate_;
 325   transport::VideoCodec codec_;
 326   uint32 last_encoded_frame_id_;
 327
 328   // Shared memory buffers for input/output with the VideoAccelerator.
 329   ScopedVector<base::SharedMemory> input_buffers_;
 330   ScopedVector<base::SharedMemory> output_buffers_;
 331
 332   // Input buffers ready to be filled with input from Encode(). As a LIFO since
 333   // we don't care about ordering.
 334   std::vector<int> input_buffers_free_;
 335
 336   // FIFO list.
 337   std::list<EncodedFrameReturnData> encoded_frame_data_storage_;
 338
 339   DISALLOW_COPY_AND_ASSIGN(LocalVideoEncodeAcceleratorClient);
 340 };
 341
 342 ExternalVideoEncoder::ExternalVideoEncoder(
 343     scoped_refptr<CastEnvironment> cast_environment,
 344     const VideoSenderConfig& video_config,
 345     scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories)
 346     : video_config_(video_config),
 347       cast_environment_(cast_environment),
 348       encoder_active_(false),
 349       key_frame_requested_(false),
 350       skip_next_frame_(false),
 351       skip_count_(0),
 352       encoder_task_runner_(gpu_factories->GetTaskRunner()),
 353       weak_factory_(this)  {
 354   DCHECK(gpu_factories);
 355   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 356   video_accelerator_client_ = new LocalVideoEncodeAcceleratorClient(
 357       cast_environment, gpu_factories, weak_factory_.GetWeakPtr());
 358
 359   encoder_task_runner_->PostTask(
 360       FROM_HERE,
 361       base::Bind(&LocalVideoEncodeAcceleratorClient::Initialize,
 362       video_accelerator_client_, video_config));
 363 }
 364
 365 ExternalVideoEncoder::~ExternalVideoEncoder() {
 366   encoder_task_runner_->PostTask(
 367       FROM_HERE,
 368       base::Bind(&LocalVideoEncodeAcceleratorClient::Destroy,
 369       video_accelerator_client_));
 370 }
 371
 372 void ExternalVideoEncoder::EncoderInitialized() {
 373   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 374   encoder_active_ = true;
 375 }
 376
 377 void ExternalVideoEncoder::EncoderError() {
 378   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 379   encoder_active_ = false;
 380 }
 381
 382 bool ExternalVideoEncoder::EncodeVideoFrame(
 383     const scoped_refptr<media::VideoFrame>& video_frame,
 384     const base::TimeTicks& capture_time,
 385     const FrameEncodedCallback& frame_encoded_callback) {
 386   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 387
 388   if (!encoder_active_) return false;
 389
 390   if (skip_next_frame_) {
 391     VLOG(1) << "Skip encoding frame";
 392     ++skip_count_;
 393     skip_next_frame_ = false;
 394     return false;
 395   }
 396   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 397   cast_environment_->Logging()->InsertFrameEvent(now, kVideoFrameSentToEncoder,
 398       GetVideoRtpTimestamp(capture_time), kFrameIdUnknown);
 399
 400   encoder_task_runner_->PostTask(
 401       FROM_HERE,
 402       base::Bind(&LocalVideoEncodeAcceleratorClient::EncodeVideoFrame,
 403       video_accelerator_client_, video_frame, capture_time,
 404       key_frame_requested_, frame_encoded_callback));
 405
 406   key_frame_requested_ = false;
 407   return true;
 408 }
 409
 410 // Inform the encoder about the new target bit rate.
 411 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 412   encoder_task_runner_->PostTask(
 413       FROM_HERE,
 414       base::Bind(&LocalVideoEncodeAcceleratorClient::SetBitRate,
 415       video_accelerator_client_, new_bit_rate));
 416 }
 417
 418 // Inform the encoder to not encode the next frame.
 419 void ExternalVideoEncoder::SkipNextFrame(bool skip_next_frame) {
 420   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 421   skip_next_frame_ = skip_next_frame;
 422 }
 423
 424 // Inform the encoder to encode the next frame as a key frame.
 425 void ExternalVideoEncoder::GenerateKeyFrame() {
 426   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 427   key_frame_requested_ = true;
 428 }
 429
 430 // Inform the encoder to only reference frames older or equal to frame_id;
 431 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 432   // Do nothing not supported.
 433 }
 434
 435 int ExternalVideoEncoder::NumberOfSkippedFrames() const {
 436   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 437   return skip_count_;
 438 }
 439
 440 }  //  namespace cast
 441 }  //  namespace media