media/cast/sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/base/video_util.h"
  15 #include "media/cast/cast_defines.h"
  16 #include "media/cast/logging/logging_defines.h"
  17 #include "media/cast/net/cast_transport_config.h"
  18
  19 namespace {
  20
  21 static const size_t kOutputBufferCount = 3;
  22
  23 void LogFrameEncodedEvent(
  24     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  25     base::TimeTicks event_time,
  26     media::cast::RtpTimestamp rtp_timestamp,
  27     uint32 frame_id) {
  28   cast_environment->Logging()->InsertFrameEvent(
  29       event_time, media::cast::FRAME_ENCODED, media::cast::VIDEO_EVENT,
  30       rtp_timestamp, frame_id);
  31 }
  32
  33 }  // namespace
  34
  35 namespace media {
  36 namespace cast {
  37
  38 // Container for the associated data of a video frame being processed.
  39 struct InProgressFrameEncode {
  40   const RtpTimestamp rtp_timestamp;
  41   const base::TimeTicks reference_time;
  42   const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  43
  44   InProgressFrameEncode(RtpTimestamp rtp,
  45                         base::TimeTicks r_time,
  46                         VideoEncoder::FrameEncodedCallback callback)
  47       : rtp_timestamp(rtp),
  48         reference_time(r_time),
  49         frame_encoded_callback(callback) {}
  50 };
  51
  52 // Owns a VideoEncoderAccelerator instance and provides the necessary adapters
  53 // to encode media::VideoFrames and emit media::cast::EncodedFrames.  All
  54 // methods must be called on the thread associated with the given
  55 // SingleThreadTaskRunner, except for the task_runner() accessor.
  56 class ExternalVideoEncoder::VEAClientImpl
  57     : public VideoEncodeAccelerator::Client,
  58       public base::RefCountedThreadSafe<VEAClientImpl> {
  59  public:
  60   VEAClientImpl(
  61       const scoped_refptr<CastEnvironment>& cast_environment,
  62       const scoped_refptr<base::SingleThreadTaskRunner>& encoder_task_runner,
  63       scoped_ptr<media::VideoEncodeAccelerator> vea,
  64       int max_frame_rate,
  65       const StatusChangeCallback& status_change_cb,
  66       const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
  67       : cast_environment_(cast_environment),
  68         task_runner_(encoder_task_runner),
  69         max_frame_rate_(max_frame_rate),
  70         status_change_cb_(status_change_cb),
  71         create_video_encode_memory_cb_(create_video_encode_memory_cb),
  72         video_encode_accelerator_(vea.Pass()),
  73         encoder_active_(false),
  74         next_frame_id_(0u),
  75         key_frame_encountered_(false) {
  76   }
  77
  78   base::SingleThreadTaskRunner* task_runner() const {
  79     return task_runner_.get();
  80   }
  81
  82   void Initialize(const gfx::Size& frame_size,
  83                   VideoCodecProfile codec_profile,
  84                   int start_bit_rate,
  85                   uint32 first_frame_id) {
  86     DCHECK(task_runner_->RunsTasksOnCurrentThread());
  87
  88     encoder_active_ = video_encode_accelerator_->Initialize(
  89         media::PIXEL_FORMAT_I420, frame_size, codec_profile, start_bit_rate,
  90         this);
  91     next_frame_id_ = first_frame_id;
  92
  93     UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
  94                           encoder_active_);
  95
  96     cast_environment_->PostTask(
  97         CastEnvironment::MAIN,
  98         FROM_HERE,
  99         base::Bind(status_change_cb_,
 100                    encoder_active_ ? STATUS_INITIALIZED :
 101                        STATUS_CODEC_INIT_FAILED));
 102   }
 103
 104   void SetBitRate(int bit_rate) {
 105     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 106
 107     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 108                                                                max_frame_rate_);
 109   }
 110
 111   void EncodeVideoFrame(
 112       const scoped_refptr<media::VideoFrame>& video_frame,
 113       const base::TimeTicks& reference_time,
 114       bool key_frame_requested,
 115       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 116     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 117
 118     if (!encoder_active_)
 119       return;
 120
 121     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
 122         TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency),
 123         reference_time,
 124         frame_encoded_callback));
 125
 126     // BitstreamBufferReady will be called once the encoder is done.
 127     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 128   }
 129
 130  protected:
 131   void NotifyError(VideoEncodeAccelerator::Error error) final {
 132     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 133
 134     DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
 135            error != VideoEncodeAccelerator::kIllegalStateError);
 136
 137     encoder_active_ = false;
 138
 139     cast_environment_->PostTask(
 140         CastEnvironment::MAIN,
 141         FROM_HERE,
 142         base::Bind(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 143
 144     // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
 145     // pending frames do not become stuck, freezing VideoSender.
 146   }
 147
 148   // Called to allocate the input and output buffers.
 149   void RequireBitstreamBuffers(unsigned int input_count,
 150                                const gfx::Size& input_coded_size,
 151                                size_t output_buffer_size) final {
 152     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 153
 154     // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
 155     // using |kOutputBufferCount| (3) here.
 156     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 157       create_video_encode_memory_cb_.Run(
 158           output_buffer_size,
 159           base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
 160     }
 161   }
 162
 163   // Encoder has encoded a frame and it's available in one of the output
 164   // buffers.  Package the result in a media::cast::EncodedFrame and post it
 165   // to the Cast MAIN thread via the supplied callback.
 166   void BitstreamBufferReady(int32 bitstream_buffer_id,
 167                             size_t payload_size,
 168                             bool key_frame) final {
 169     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 170     if (bitstream_buffer_id < 0 ||
 171         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 172       NOTREACHED();
 173       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 174               << bitstream_buffer_id;
 175       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 176       return;
 177     }
 178     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 179     if (payload_size > output_buffer->mapped_size()) {
 180       NOTREACHED();
 181       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 182               << payload_size;
 183       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 184       return;
 185     }
 186     if (key_frame)
 187       key_frame_encountered_ = true;
 188     if (!key_frame_encountered_) {
 189       // Do not send video until we have encountered the first key frame.
 190       // Save the bitstream buffer in |stream_header_| to be sent later along
 191       // with the first key frame.
 192       //
 193       // TODO(miu): Should |stream_header_| be an std::ostringstream for
 194       // performance reasons?
 195       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 196                             payload_size);
 197     } else if (!in_progress_frame_encodes_.empty()) {
 198       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 199
 200       scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
 201       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
 202           EncodedFrame::DEPENDENT;
 203       encoded_frame->frame_id = next_frame_id_++;
 204       if (key_frame)
 205         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 206       else
 207         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
 208       encoded_frame->rtp_timestamp = request.rtp_timestamp;
 209       encoded_frame->reference_time = request.reference_time;
 210       if (!stream_header_.empty()) {
 211         encoded_frame->data = stream_header_;
 212         stream_header_.clear();
 213       }
 214       encoded_frame->data.append(
 215           static_cast<const char*>(output_buffer->memory()), payload_size);
 216       // TODO(miu): Compute and populate the |deadline_utilization| and
 217       // |lossy_utilization| performance metrics in |encoded_frame|.
 218
 219       cast_environment_->PostTask(
 220           CastEnvironment::MAIN,
 221           FROM_HERE,
 222           base::Bind(&LogFrameEncodedEvent,
 223                      cast_environment_,
 224                      cast_environment_->Clock()->NowTicks(),
 225                      encoded_frame->rtp_timestamp,
 226                      encoded_frame->frame_id));
 227
 228       cast_environment_->PostTask(
 229           CastEnvironment::MAIN,
 230           FROM_HERE,
 231           base::Bind(request.frame_encoded_callback,
 232                      base::Passed(&encoded_frame)));
 233
 234       in_progress_frame_encodes_.pop_front();
 235     } else {
 236       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 237     }
 238
 239     // We need to re-add the output buffer to the encoder after we are done
 240     // with it.
 241     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 242         bitstream_buffer_id,
 243         output_buffers_[bitstream_buffer_id]->handle(),
 244         output_buffers_[bitstream_buffer_id]->mapped_size()));
 245   }
 246
 247  private:
 248   friend class base::RefCountedThreadSafe<VEAClientImpl>;
 249
 250   ~VEAClientImpl() final {
 251     // According to the media::VideoEncodeAccelerator interface, Destroy()
 252     // should be called instead of invoking its private destructor.
 253     task_runner_->PostTask(
 254         FROM_HERE,
 255         base::Bind(&media::VideoEncodeAccelerator::Destroy,
 256                    base::Unretained(video_encode_accelerator_.release())));
 257   }
 258
 259   // Note: This method can be called on any thread.
 260   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 261     task_runner_->PostTask(FROM_HERE,
 262                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
 263                                       this,
 264                                       base::Passed(&memory)));
 265   }
 266
 267   void OnReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 268     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 269
 270     output_buffers_.push_back(memory.Pass());
 271
 272     // Wait until all requested buffers are received.
 273     if (output_buffers_.size() < kOutputBufferCount)
 274       return;
 275
 276     // Immediately provide all output buffers to the VEA.
 277     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 278       video_encode_accelerator_->UseOutputBitstreamBuffer(
 279           media::BitstreamBuffer(static_cast<int32>(i),
 280                                  output_buffers_[i]->handle(),
 281                                  output_buffers_[i]->mapped_size()));
 282     }
 283   }
 284
 285   const scoped_refptr<CastEnvironment> cast_environment_;
 286   const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 287   const int max_frame_rate_;
 288   const StatusChangeCallback status_change_cb_;  // Must be run on MAIN thread.
 289   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 290   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 291   bool encoder_active_;
 292   uint32 next_frame_id_;
 293   bool key_frame_encountered_;
 294   std::string stream_header_;
 295
 296   // Shared memory buffers for output with the VideoAccelerator.
 297   ScopedVector<base::SharedMemory> output_buffers_;
 298
 299   // FIFO list.
 300   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 301
 302   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 303 };
 304
 305 // static
 306 bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) {
 307   if (video_config.codec != CODEC_VIDEO_VP8 &&
 308       video_config.codec != CODEC_VIDEO_H264)
 309     return false;
 310
 311   // TODO(miu): "Layering hooks" are needed to be able to query outside of
 312   // libmedia, to determine whether the system provides a hardware encoder.  For
 313   // now, assume that this was already checked by this point.
 314   // http://crbug.com/454029
 315   return video_config.use_external_encoder;
 316 }
 317
 318 ExternalVideoEncoder::ExternalVideoEncoder(
 319     const scoped_refptr<CastEnvironment>& cast_environment,
 320     const VideoSenderConfig& video_config,
 321     const gfx::Size& frame_size,
 322     uint32 first_frame_id,
 323     const StatusChangeCallback& status_change_cb,
 324     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 325     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 326     : cast_environment_(cast_environment),
 327       create_video_encode_memory_cb_(create_video_encode_memory_cb),
 328       frame_size_(frame_size),
 329       bit_rate_(video_config.start_bitrate),
 330       key_frame_requested_(false),
 331       weak_factory_(this) {
 332   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 333   DCHECK_GT(video_config.max_frame_rate, 0);
 334   DCHECK(!frame_size_.IsEmpty());
 335   DCHECK(!status_change_cb.is_null());
 336   DCHECK(!create_vea_cb.is_null());
 337   DCHECK(!create_video_encode_memory_cb_.is_null());
 338   DCHECK_GT(bit_rate_, 0);
 339
 340   create_vea_cb.Run(
 341       base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
 342                  weak_factory_.GetWeakPtr(),
 343                  video_config,
 344                  first_frame_id,
 345                  status_change_cb));
 346 }
 347
 348 ExternalVideoEncoder::~ExternalVideoEncoder() {
 349 }
 350
 351 bool ExternalVideoEncoder::EncodeVideoFrame(
 352     const scoped_refptr<media::VideoFrame>& video_frame,
 353     const base::TimeTicks& reference_time,
 354     const FrameEncodedCallback& frame_encoded_callback) {
 355   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 356   DCHECK(!frame_encoded_callback.is_null());
 357
 358   if (!client_ || video_frame->visible_rect().size() != frame_size_)
 359     return false;
 360
 361   client_->task_runner()->PostTask(FROM_HERE,
 362                                    base::Bind(&VEAClientImpl::EncodeVideoFrame,
 363                                               client_,
 364                                               video_frame,
 365                                               reference_time,
 366                                               key_frame_requested_,
 367                                               frame_encoded_callback));
 368   key_frame_requested_ = false;
 369   return true;
 370 }
 371
 372 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 373   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 374   DCHECK_GT(new_bit_rate, 0);
 375
 376   bit_rate_ = new_bit_rate;
 377   if (!client_)
 378     return;
 379   client_->task_runner()->PostTask(
 380       FROM_HERE, base::Bind(&VEAClientImpl::SetBitRate, client_, bit_rate_));
 381 }
 382
 383 void ExternalVideoEncoder::GenerateKeyFrame() {
 384   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 385   key_frame_requested_ = true;
 386 }
 387
 388 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 389   // Do nothing.  Not supported.
 390 }
 391
 392 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 393     const VideoSenderConfig& video_config,
 394     uint32 first_frame_id,
 395     const StatusChangeCallback& status_change_cb,
 396     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 397     scoped_ptr<media::VideoEncodeAccelerator> vea) {
 398   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 399
 400   // The callback will be invoked with null pointers in the case where the
 401   // system does not support or lacks the resources to provide GPU-accelerated
 402   // video encoding.
 403   if (!encoder_task_runner || !vea) {
 404     cast_environment_->PostTask(
 405         CastEnvironment::MAIN,
 406         FROM_HERE,
 407         base::Bind(status_change_cb, STATUS_CODEC_INIT_FAILED));
 408     return;
 409   }
 410
 411   VideoCodecProfile codec_profile;
 412   switch (video_config.codec) {
 413     case CODEC_VIDEO_VP8:
 414       codec_profile = media::VP8PROFILE_ANY;
 415       break;
 416     case CODEC_VIDEO_H264:
 417       codec_profile = media::H264PROFILE_MAIN;
 418       break;
 419     case CODEC_VIDEO_FAKE:
 420       NOTREACHED() << "Fake software video encoder cannot be external";
 421       // ...flow through to next case...
 422     default:
 423       cast_environment_->PostTask(
 424         CastEnvironment::MAIN,
 425         FROM_HERE,
 426         base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
 427       return;
 428   }
 429
 430   DCHECK(!client_);
 431   client_ = new VEAClientImpl(cast_environment_,
 432                               encoder_task_runner,
 433                               vea.Pass(),
 434                               video_config.max_frame_rate,
 435                               status_change_cb,
 436                               create_video_encode_memory_cb_);
 437   client_->task_runner()->PostTask(FROM_HERE,
 438                                    base::Bind(&VEAClientImpl::Initialize,
 439                                               client_,
 440                                               frame_size_,
 441                                               codec_profile,
 442                                               bit_rate_,
 443                                               first_frame_id));
 444 }
 445
 446 SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder(
 447     const scoped_refptr<CastEnvironment>& cast_environment,
 448     const VideoSenderConfig& video_config,
 449     const StatusChangeCallback& status_change_cb,
 450     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 451     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 452     : SizeAdaptableVideoEncoderBase(cast_environment,
 453                                     video_config,
 454                                     status_change_cb),
 455       create_vea_cb_(create_vea_cb),
 456       create_video_encode_memory_cb_(create_video_encode_memory_cb) {}
 457
 458 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {}
 459
 460 scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() {
 461   return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder(
 462       cast_environment(),
 463       video_config(),
 464       frame_size(),
 465       last_frame_id() + 1,
 466       CreateEncoderStatusChangeCallback(),
 467       create_vea_cb_,
 468       create_video_encode_memory_cb_));
 469 }
 470
 471 }  //  namespace cast
 472 }  //  namespace media