media/cast/sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/base/video_util.h"
  15 #include "media/cast/cast_defines.h"
  16 #include "media/cast/logging/logging_defines.h"
  17 #include "media/cast/net/cast_transport_config.h"
  18
  19 namespace {
  20
  21 static const size_t kOutputBufferCount = 3;
  22
  23 void LogFrameEncodedEvent(
  24     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  25     base::TimeTicks event_time,
  26     media::cast::RtpTimestamp rtp_timestamp,
  27     uint32 frame_id) {
  28   cast_environment->Logging()->InsertFrameEvent(
  29       event_time, media::cast::FRAME_ENCODED, media::cast::VIDEO_EVENT,
  30       rtp_timestamp, frame_id);
  31 }
  32
  33 }  // namespace
  34
  35 namespace media {
  36 namespace cast {
  37
  38 // Container for the associated data of a video frame being processed.
  39 struct InProgressFrameEncode {
  40   const RtpTimestamp rtp_timestamp;
  41   const base::TimeTicks reference_time;
  42   const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  43
  44   InProgressFrameEncode(RtpTimestamp rtp,
  45                         base::TimeTicks r_time,
  46                         VideoEncoder::FrameEncodedCallback callback)
  47       : rtp_timestamp(rtp),
  48         reference_time(r_time),
  49         frame_encoded_callback(callback) {}
  50 };
  51
  52 // Owns a VideoEncoderAccelerator instance and provides the necessary adapters
  53 // to encode media::VideoFrames and emit media::cast::EncodedFrames.  All
  54 // methods must be called on the thread associated with the given
  55 // SingleThreadTaskRunner, except for the task_runner() accessor.
  56 class ExternalVideoEncoder::VEAClientImpl
  57     : public VideoEncodeAccelerator::Client,
  58       public base::RefCountedThreadSafe<VEAClientImpl> {
  59  public:
  60   VEAClientImpl(
  61       const scoped_refptr<CastEnvironment>& cast_environment,
  62       const scoped_refptr<base::SingleThreadTaskRunner>& encoder_task_runner,
  63       scoped_ptr<media::VideoEncodeAccelerator> vea,
  64       int max_frame_rate,
  65       const StatusChangeCallback& status_change_cb,
  66       const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
  67       : cast_environment_(cast_environment),
  68         task_runner_(encoder_task_runner),
  69         max_frame_rate_(max_frame_rate),
  70         status_change_cb_(status_change_cb),
  71         create_video_encode_memory_cb_(create_video_encode_memory_cb),
  72         video_encode_accelerator_(vea.Pass()),
  73         encoder_active_(false),
  74         next_frame_id_(0u),
  75         key_frame_encountered_(false) {
  76   }
  77
  78   base::SingleThreadTaskRunner* task_runner() const {
  79     return task_runner_.get();
  80   }
  81
  82   void Initialize(const gfx::Size& frame_size,
  83                   VideoCodecProfile codec_profile,
  84                   int start_bit_rate,
  85                   uint32 first_frame_id) {
  86     DCHECK(task_runner_->RunsTasksOnCurrentThread());
  87
  88     encoder_active_ = video_encode_accelerator_->Initialize(
  89         media::VideoFrame::I420,
  90         frame_size,
  91         codec_profile,
  92         start_bit_rate,
  93         this);
  94     next_frame_id_ = first_frame_id;
  95
  96     UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
  97                           encoder_active_);
  98
  99     cast_environment_->PostTask(
 100         CastEnvironment::MAIN,
 101         FROM_HERE,
 102         base::Bind(status_change_cb_,
 103                    encoder_active_ ? STATUS_INITIALIZED :
 104                        STATUS_CODEC_INIT_FAILED));
 105   }
 106
 107   void SetBitRate(int bit_rate) {
 108     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 109
 110     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 111                                                                max_frame_rate_);
 112   }
 113
 114   void EncodeVideoFrame(
 115       const scoped_refptr<media::VideoFrame>& video_frame,
 116       const base::TimeTicks& reference_time,
 117       bool key_frame_requested,
 118       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 119     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 120
 121     if (!encoder_active_)
 122       return;
 123
 124     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
 125         TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency),
 126         reference_time,
 127         frame_encoded_callback));
 128
 129     // BitstreamBufferReady will be called once the encoder is done.
 130     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 131   }
 132
 133  protected:
 134   void NotifyError(VideoEncodeAccelerator::Error error) final {
 135     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 136
 137     DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
 138            error != VideoEncodeAccelerator::kIllegalStateError);
 139
 140     encoder_active_ = false;
 141
 142     cast_environment_->PostTask(
 143         CastEnvironment::MAIN,
 144         FROM_HERE,
 145         base::Bind(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 146
 147     // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
 148     // pending frames do not become stuck, freezing VideoSender.
 149   }
 150
 151   // Called to allocate the input and output buffers.
 152   void RequireBitstreamBuffers(unsigned int input_count,
 153                                const gfx::Size& input_coded_size,
 154                                size_t output_buffer_size) final {
 155     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 156
 157     // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
 158     // using |kOutputBufferCount| (3) here.
 159     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 160       create_video_encode_memory_cb_.Run(
 161           output_buffer_size,
 162           base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
 163     }
 164   }
 165
 166   // Encoder has encoded a frame and it's available in one of the output
 167   // buffers.  Package the result in a media::cast::EncodedFrame and post it
 168   // to the Cast MAIN thread via the supplied callback.
 169   void BitstreamBufferReady(int32 bitstream_buffer_id,
 170                             size_t payload_size,
 171                             bool key_frame) final {
 172     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 173     if (bitstream_buffer_id < 0 ||
 174         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 175       NOTREACHED();
 176       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 177               << bitstream_buffer_id;
 178       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 179       return;
 180     }
 181     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 182     if (payload_size > output_buffer->mapped_size()) {
 183       NOTREACHED();
 184       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 185               << payload_size;
 186       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 187       return;
 188     }
 189     if (key_frame)
 190       key_frame_encountered_ = true;
 191     if (!key_frame_encountered_) {
 192       // Do not send video until we have encountered the first key frame.
 193       // Save the bitstream buffer in |stream_header_| to be sent later along
 194       // with the first key frame.
 195       //
 196       // TODO(miu): Should |stream_header_| be an std::ostringstream for
 197       // performance reasons?
 198       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 199                             payload_size);
 200     } else if (!in_progress_frame_encodes_.empty()) {
 201       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 202
 203       scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
 204       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
 205           EncodedFrame::DEPENDENT;
 206       encoded_frame->frame_id = next_frame_id_++;
 207       if (key_frame)
 208         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 209       else
 210         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
 211       encoded_frame->rtp_timestamp = request.rtp_timestamp;
 212       encoded_frame->reference_time = request.reference_time;
 213       if (!stream_header_.empty()) {
 214         encoded_frame->data = stream_header_;
 215         stream_header_.clear();
 216       }
 217       encoded_frame->data.append(
 218           static_cast<const char*>(output_buffer->memory()), payload_size);
 219       // TODO(miu): Compute and populate the |deadline_utilization| and
 220       // |lossy_utilization| performance metrics in |encoded_frame|.
 221
 222       cast_environment_->PostTask(
 223           CastEnvironment::MAIN,
 224           FROM_HERE,
 225           base::Bind(&LogFrameEncodedEvent,
 226                      cast_environment_,
 227                      cast_environment_->Clock()->NowTicks(),
 228                      encoded_frame->rtp_timestamp,
 229                      encoded_frame->frame_id));
 230
 231       cast_environment_->PostTask(
 232           CastEnvironment::MAIN,
 233           FROM_HERE,
 234           base::Bind(request.frame_encoded_callback,
 235                      base::Passed(&encoded_frame)));
 236
 237       in_progress_frame_encodes_.pop_front();
 238     } else {
 239       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 240     }
 241
 242     // We need to re-add the output buffer to the encoder after we are done
 243     // with it.
 244     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 245         bitstream_buffer_id,
 246         output_buffers_[bitstream_buffer_id]->handle(),
 247         output_buffers_[bitstream_buffer_id]->mapped_size()));
 248   }
 249
 250  private:
 251   friend class base::RefCountedThreadSafe<VEAClientImpl>;
 252
 253   ~VEAClientImpl() final {
 254     // According to the media::VideoEncodeAccelerator interface, Destroy()
 255     // should be called instead of invoking its private destructor.
 256     task_runner_->PostTask(
 257         FROM_HERE,
 258         base::Bind(&media::VideoEncodeAccelerator::Destroy,
 259                    base::Unretained(video_encode_accelerator_.release())));
 260   }
 261
 262   // Note: This method can be called on any thread.
 263   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 264     task_runner_->PostTask(FROM_HERE,
 265                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
 266                                       this,
 267                                       base::Passed(&memory)));
 268   }
 269
 270   void OnReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 271     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 272
 273     output_buffers_.push_back(memory.Pass());
 274
 275     // Wait until all requested buffers are received.
 276     if (output_buffers_.size() < kOutputBufferCount)
 277       return;
 278
 279     // Immediately provide all output buffers to the VEA.
 280     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 281       video_encode_accelerator_->UseOutputBitstreamBuffer(
 282           media::BitstreamBuffer(static_cast<int32>(i),
 283                                  output_buffers_[i]->handle(),
 284                                  output_buffers_[i]->mapped_size()));
 285     }
 286   }
 287
 288   const scoped_refptr<CastEnvironment> cast_environment_;
 289   const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 290   const int max_frame_rate_;
 291   const StatusChangeCallback status_change_cb_;  // Must be run on MAIN thread.
 292   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 293   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 294   bool encoder_active_;
 295   uint32 next_frame_id_;
 296   bool key_frame_encountered_;
 297   std::string stream_header_;
 298
 299   // Shared memory buffers for output with the VideoAccelerator.
 300   ScopedVector<base::SharedMemory> output_buffers_;
 301
 302   // FIFO list.
 303   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 304
 305   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 306 };
 307
 308 // static
 309 bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) {
 310   if (video_config.codec != CODEC_VIDEO_VP8 &&
 311       video_config.codec != CODEC_VIDEO_H264)
 312     return false;
 313
 314   // TODO(miu): "Layering hooks" are needed to be able to query outside of
 315   // libmedia, to determine whether the system provides a hardware encoder.  For
 316   // now, assume that this was already checked by this point.
 317   // http://crbug.com/454029
 318   return video_config.use_external_encoder;
 319 }
 320
 321 ExternalVideoEncoder::ExternalVideoEncoder(
 322     const scoped_refptr<CastEnvironment>& cast_environment,
 323     const VideoSenderConfig& video_config,
 324     const gfx::Size& frame_size,
 325     uint32 first_frame_id,
 326     const StatusChangeCallback& status_change_cb,
 327     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 328     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 329     : cast_environment_(cast_environment),
 330       create_video_encode_memory_cb_(create_video_encode_memory_cb),
 331       frame_size_(frame_size),
 332       bit_rate_(video_config.start_bitrate),
 333       key_frame_requested_(false),
 334       weak_factory_(this) {
 335   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 336   DCHECK_GT(video_config.max_frame_rate, 0);
 337   DCHECK(!frame_size_.IsEmpty());
 338   DCHECK(!status_change_cb.is_null());
 339   DCHECK(!create_vea_cb.is_null());
 340   DCHECK(!create_video_encode_memory_cb_.is_null());
 341   DCHECK_GT(bit_rate_, 0);
 342
 343   create_vea_cb.Run(
 344       base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
 345                  weak_factory_.GetWeakPtr(),
 346                  video_config,
 347                  first_frame_id,
 348                  status_change_cb));
 349 }
 350
 351 ExternalVideoEncoder::~ExternalVideoEncoder() {
 352 }
 353
 354 bool ExternalVideoEncoder::EncodeVideoFrame(
 355     const scoped_refptr<media::VideoFrame>& video_frame,
 356     const base::TimeTicks& reference_time,
 357     const FrameEncodedCallback& frame_encoded_callback) {
 358   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 359   DCHECK(!frame_encoded_callback.is_null());
 360
 361   if (!client_ || video_frame->visible_rect().size() != frame_size_)
 362     return false;
 363
 364   client_->task_runner()->PostTask(FROM_HERE,
 365                                    base::Bind(&VEAClientImpl::EncodeVideoFrame,
 366                                               client_,
 367                                               video_frame,
 368                                               reference_time,
 369                                               key_frame_requested_,
 370                                               frame_encoded_callback));
 371   key_frame_requested_ = false;
 372   return true;
 373 }
 374
 375 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 376   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 377   DCHECK_GT(new_bit_rate, 0);
 378
 379   bit_rate_ = new_bit_rate;
 380   if (!client_)
 381     return;
 382   client_->task_runner()->PostTask(
 383       FROM_HERE, base::Bind(&VEAClientImpl::SetBitRate, client_, bit_rate_));
 384 }
 385
 386 void ExternalVideoEncoder::GenerateKeyFrame() {
 387   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 388   key_frame_requested_ = true;
 389 }
 390
 391 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 392   // Do nothing.  Not supported.
 393 }
 394
 395 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 396     const VideoSenderConfig& video_config,
 397     uint32 first_frame_id,
 398     const StatusChangeCallback& status_change_cb,
 399     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 400     scoped_ptr<media::VideoEncodeAccelerator> vea) {
 401   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 402
 403   // The callback will be invoked with null pointers in the case where the
 404   // system does not support or lacks the resources to provide GPU-accelerated
 405   // video encoding.
 406   if (!encoder_task_runner || !vea) {
 407     cast_environment_->PostTask(
 408         CastEnvironment::MAIN,
 409         FROM_HERE,
 410         base::Bind(status_change_cb, STATUS_CODEC_INIT_FAILED));
 411     return;
 412   }
 413
 414   VideoCodecProfile codec_profile;
 415   switch (video_config.codec) {
 416     case CODEC_VIDEO_VP8:
 417       codec_profile = media::VP8PROFILE_ANY;
 418       break;
 419     case CODEC_VIDEO_H264:
 420       codec_profile = media::H264PROFILE_MAIN;
 421       break;
 422     case CODEC_VIDEO_FAKE:
 423       NOTREACHED() << "Fake software video encoder cannot be external";
 424       // ...flow through to next case...
 425     default:
 426       cast_environment_->PostTask(
 427         CastEnvironment::MAIN,
 428         FROM_HERE,
 429         base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
 430       return;
 431   }
 432
 433   DCHECK(!client_);
 434   client_ = new VEAClientImpl(cast_environment_,
 435                               encoder_task_runner,
 436                               vea.Pass(),
 437                               video_config.max_frame_rate,
 438                               status_change_cb,
 439                               create_video_encode_memory_cb_);
 440   client_->task_runner()->PostTask(FROM_HERE,
 441                                    base::Bind(&VEAClientImpl::Initialize,
 442                                               client_,
 443                                               frame_size_,
 444                                               codec_profile,
 445                                               bit_rate_,
 446                                               first_frame_id));
 447 }
 448
 449 SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder(
 450     const scoped_refptr<CastEnvironment>& cast_environment,
 451     const VideoSenderConfig& video_config,
 452     const StatusChangeCallback& status_change_cb,
 453     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 454     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 455     : SizeAdaptableVideoEncoderBase(cast_environment,
 456                                     video_config,
 457                                     status_change_cb),
 458       create_vea_cb_(create_vea_cb),
 459       create_video_encode_memory_cb_(create_video_encode_memory_cb) {}
 460
 461 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {}
 462
 463 scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() {
 464   return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder(
 465       cast_environment(),
 466       video_config(),
 467       frame_size(),
 468       last_frame_id() + 1,
 469       CreateEncoderStatusChangeCallback(),
 470       create_vea_cb_,
 471       create_video_encode_memory_cb_));
 472 }
 473
 474 }  //  namespace cast
 475 }  //  namespace media