media/cast/sender/external_video_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/external_video_encoder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_vector.h"
  10 #include "base/memory/shared_memory.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "base/metrics/histogram.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/base/video_util.h"
  15 #include "media/cast/cast_defines.h"
  16 #include "media/cast/logging/logging_defines.h"
  17 #include "media/cast/net/cast_transport_config.h"
  18 #include "media/video/video_encode_accelerator.h"
  19
  20 namespace {
  21
  22 static const size_t kOutputBufferCount = 3;
  23
  24 void LogFrameEncodedEvent(
  25     const scoped_refptr<media::cast::CastEnvironment>& cast_environment,
  26     base::TimeTicks event_time,
  27     media::cast::RtpTimestamp rtp_timestamp,
  28     uint32 frame_id) {
  29   cast_environment->Logging()->InsertFrameEvent(
  30       event_time, media::cast::FRAME_ENCODED, media::cast::VIDEO_EVENT,
  31       rtp_timestamp, frame_id);
  32 }
  33
  34 }  // namespace
  35
  36 namespace media {
  37 namespace cast {
  38
  39 // Container for the associated data of a video frame being processed.
  40 struct InProgressFrameEncode {
  41   const RtpTimestamp rtp_timestamp;
  42   const base::TimeTicks reference_time;
  43   const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
  44
  45   InProgressFrameEncode(RtpTimestamp rtp,
  46                         base::TimeTicks r_time,
  47                         VideoEncoder::FrameEncodedCallback callback)
  48       : rtp_timestamp(rtp),
  49         reference_time(r_time),
  50         frame_encoded_callback(callback) {}
  51 };
  52
  53 // Owns a VideoEncoderAccelerator instance and provides the necessary adapters
  54 // to encode media::VideoFrames and emit media::cast::EncodedFrames.  All
  55 // methods must be called on the thread associated with the given
  56 // SingleThreadTaskRunner, except for the task_runner() accessor.
  57 class ExternalVideoEncoder::VEAClientImpl
  58     : public VideoEncodeAccelerator::Client,
  59       public base::RefCountedThreadSafe<VEAClientImpl> {
  60  public:
  61   VEAClientImpl(
  62       const scoped_refptr<CastEnvironment>& cast_environment,
  63       const scoped_refptr<base::SingleThreadTaskRunner>& encoder_task_runner,
  64       scoped_ptr<media::VideoEncodeAccelerator> vea,
  65       int max_frame_rate,
  66       const StatusChangeCallback& status_change_cb,
  67       const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
  68       : cast_environment_(cast_environment),
  69         task_runner_(encoder_task_runner),
  70         max_frame_rate_(max_frame_rate),
  71         status_change_cb_(status_change_cb),
  72         create_video_encode_memory_cb_(create_video_encode_memory_cb),
  73         video_encode_accelerator_(vea.Pass()),
  74         encoder_active_(false),
  75         next_frame_id_(0u),
  76         key_frame_encountered_(false) {
  77   }
  78
  79   base::SingleThreadTaskRunner* task_runner() const {
  80     return task_runner_.get();
  81   }
  82
  83   void Initialize(const gfx::Size& frame_size,
  84                   VideoCodecProfile codec_profile,
  85                   int start_bit_rate,
  86                   uint32 first_frame_id) {
  87     DCHECK(task_runner_->RunsTasksOnCurrentThread());
  88
  89     encoder_active_ = video_encode_accelerator_->Initialize(
  90         media::VideoFrame::I420,
  91         frame_size,
  92         codec_profile,
  93         start_bit_rate,
  94         this);
  95     next_frame_id_ = first_frame_id;
  96
  97     UMA_HISTOGRAM_BOOLEAN("Cast.Sender.VideoEncodeAcceleratorInitializeSuccess",
  98                           encoder_active_);
  99
 100     cast_environment_->PostTask(
 101         CastEnvironment::MAIN,
 102         FROM_HERE,
 103         base::Bind(status_change_cb_,
 104                    encoder_active_ ? STATUS_INITIALIZED :
 105                        STATUS_CODEC_INIT_FAILED));
 106   }
 107
 108   void SetBitRate(int bit_rate) {
 109     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 110
 111     video_encode_accelerator_->RequestEncodingParametersChange(bit_rate,
 112                                                                max_frame_rate_);
 113   }
 114
 115   void EncodeVideoFrame(
 116       const scoped_refptr<media::VideoFrame>& video_frame,
 117       const base::TimeTicks& reference_time,
 118       bool key_frame_requested,
 119       const VideoEncoder::FrameEncodedCallback& frame_encoded_callback) {
 120     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 121
 122     if (!encoder_active_)
 123       return;
 124
 125     in_progress_frame_encodes_.push_back(InProgressFrameEncode(
 126         TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency),
 127         reference_time,
 128         frame_encoded_callback));
 129
 130     // BitstreamBufferReady will be called once the encoder is done.
 131     video_encode_accelerator_->Encode(video_frame, key_frame_requested);
 132   }
 133
 134  protected:
 135   void NotifyError(VideoEncodeAccelerator::Error error) override {
 136     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 137
 138     DCHECK(error != VideoEncodeAccelerator::kInvalidArgumentError &&
 139            error != VideoEncodeAccelerator::kIllegalStateError);
 140
 141     encoder_active_ = false;
 142
 143     cast_environment_->PostTask(
 144         CastEnvironment::MAIN,
 145         FROM_HERE,
 146         base::Bind(status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
 147
 148     // TODO(miu): Force-flush all |in_progress_frame_encodes_| immediately so
 149     // pending frames do not become stuck, freezing VideoSender.
 150   }
 151
 152   // Called to allocate the input and output buffers.
 153   void RequireBitstreamBuffers(unsigned int input_count,
 154                                const gfx::Size& input_coded_size,
 155                                size_t output_buffer_size) override {
 156     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 157
 158     // TODO(miu): Investigate why we are ignoring |input_count| (4) and instead
 159     // using |kOutputBufferCount| (3) here.
 160     for (size_t j = 0; j < kOutputBufferCount; ++j) {
 161       create_video_encode_memory_cb_.Run(
 162           output_buffer_size,
 163           base::Bind(&VEAClientImpl::OnCreateSharedMemory, this));
 164     }
 165   }
 166
 167   // Encoder has encoded a frame and it's available in one of the output
 168   // buffers.  Package the result in a media::cast::EncodedFrame and post it
 169   // to the Cast MAIN thread via the supplied callback.
 170   void BitstreamBufferReady(int32 bitstream_buffer_id,
 171                             size_t payload_size,
 172                             bool key_frame) override {
 173     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 174     if (bitstream_buffer_id < 0 ||
 175         bitstream_buffer_id >= static_cast<int32>(output_buffers_.size())) {
 176       NOTREACHED();
 177       VLOG(1) << "BitstreamBufferReady(): invalid bitstream_buffer_id="
 178               << bitstream_buffer_id;
 179       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 180       return;
 181     }
 182     base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
 183     if (payload_size > output_buffer->mapped_size()) {
 184       NOTREACHED();
 185       VLOG(1) << "BitstreamBufferReady(): invalid payload_size = "
 186               << payload_size;
 187       NotifyError(media::VideoEncodeAccelerator::kPlatformFailureError);
 188       return;
 189     }
 190     if (key_frame)
 191       key_frame_encountered_ = true;
 192     if (!key_frame_encountered_) {
 193       // Do not send video until we have encountered the first key frame.
 194       // Save the bitstream buffer in |stream_header_| to be sent later along
 195       // with the first key frame.
 196       //
 197       // TODO(miu): Should |stream_header_| be an std::ostringstream for
 198       // performance reasons?
 199       stream_header_.append(static_cast<const char*>(output_buffer->memory()),
 200                             payload_size);
 201     } else if (!in_progress_frame_encodes_.empty()) {
 202       const InProgressFrameEncode& request = in_progress_frame_encodes_.front();
 203
 204       scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame());
 205       encoded_frame->dependency = key_frame ? EncodedFrame::KEY :
 206           EncodedFrame::DEPENDENT;
 207       encoded_frame->frame_id = next_frame_id_++;
 208       if (key_frame)
 209         encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 210       else
 211         encoded_frame->referenced_frame_id = encoded_frame->frame_id - 1;
 212       encoded_frame->rtp_timestamp = request.rtp_timestamp;
 213       encoded_frame->reference_time = request.reference_time;
 214       if (!stream_header_.empty()) {
 215         encoded_frame->data = stream_header_;
 216         stream_header_.clear();
 217       }
 218       encoded_frame->data.append(
 219           static_cast<const char*>(output_buffer->memory()), payload_size);
 220
 221       cast_environment_->PostTask(
 222           CastEnvironment::MAIN,
 223           FROM_HERE,
 224           base::Bind(&LogFrameEncodedEvent,
 225                      cast_environment_,
 226                      cast_environment_->Clock()->NowTicks(),
 227                      encoded_frame->rtp_timestamp,
 228                      encoded_frame->frame_id));
 229
 230       cast_environment_->PostTask(
 231           CastEnvironment::MAIN,
 232           FROM_HERE,
 233           base::Bind(request.frame_encoded_callback,
 234                      base::Passed(&encoded_frame)));
 235
 236       in_progress_frame_encodes_.pop_front();
 237     } else {
 238       VLOG(1) << "BitstreamBufferReady(): no encoded frame data available";
 239     }
 240
 241     // We need to re-add the output buffer to the encoder after we are done
 242     // with it.
 243     video_encode_accelerator_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
 244         bitstream_buffer_id,
 245         output_buffers_[bitstream_buffer_id]->handle(),
 246         output_buffers_[bitstream_buffer_id]->mapped_size()));
 247   }
 248
 249  private:
 250   friend class base::RefCountedThreadSafe<VEAClientImpl>;
 251
 252   ~VEAClientImpl() override {
 253     // According to the media::VideoEncodeAccelerator interface, Destroy()
 254     // should be called instead of invoking its private destructor.
 255     task_runner_->PostTask(
 256         FROM_HERE,
 257         base::Bind(&media::VideoEncodeAccelerator::Destroy,
 258                    base::Unretained(video_encode_accelerator_.release())));
 259   }
 260
 261   // Note: This method can be called on any thread.
 262   void OnCreateSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 263     task_runner_->PostTask(FROM_HERE,
 264                            base::Bind(&VEAClientImpl::OnReceivedSharedMemory,
 265                                       this,
 266                                       base::Passed(&memory)));
 267   }
 268
 269   void OnReceivedSharedMemory(scoped_ptr<base::SharedMemory> memory) {
 270     DCHECK(task_runner_->RunsTasksOnCurrentThread());
 271
 272     output_buffers_.push_back(memory.release());
 273
 274     // Wait until all requested buffers are received.
 275     if (output_buffers_.size() < kOutputBufferCount)
 276       return;
 277
 278     // Immediately provide all output buffers to the VEA.
 279     for (size_t i = 0; i < output_buffers_.size(); ++i) {
 280       video_encode_accelerator_->UseOutputBitstreamBuffer(
 281           media::BitstreamBuffer(static_cast<int32>(i),
 282                                  output_buffers_[i]->handle(),
 283                                  output_buffers_[i]->mapped_size()));
 284     }
 285   }
 286
 287   const scoped_refptr<CastEnvironment> cast_environment_;
 288   const scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 289   const int max_frame_rate_;
 290   const StatusChangeCallback status_change_cb_;  // Must be run on MAIN thread.
 291   const CreateVideoEncodeMemoryCallback create_video_encode_memory_cb_;
 292   scoped_ptr<media::VideoEncodeAccelerator> video_encode_accelerator_;
 293   bool encoder_active_;
 294   uint32 next_frame_id_;
 295   bool key_frame_encountered_;
 296   std::string stream_header_;
 297
 298   // Shared memory buffers for output with the VideoAccelerator.
 299   ScopedVector<base::SharedMemory> output_buffers_;
 300
 301   // FIFO list.
 302   std::list<InProgressFrameEncode> in_progress_frame_encodes_;
 303
 304   DISALLOW_COPY_AND_ASSIGN(VEAClientImpl);
 305 };
 306
 307 // static
 308 bool ExternalVideoEncoder::IsSupported(const VideoSenderConfig& video_config) {
 309   if (video_config.codec != CODEC_VIDEO_VP8 &&
 310       video_config.codec != CODEC_VIDEO_H264)
 311     return false;
 312
 313   // TODO(miu): "Layering hooks" are needed to be able to query outside of
 314   // libmedia, to determine whether the system provides a hardware encoder.  For
 315   // now, assume that this was already checked by this point.
 316   // http://crbug.com/454029
 317   return video_config.use_external_encoder;
 318 }
 319
 320 ExternalVideoEncoder::ExternalVideoEncoder(
 321     const scoped_refptr<CastEnvironment>& cast_environment,
 322     const VideoSenderConfig& video_config,
 323     const gfx::Size& frame_size,
 324     uint32 first_frame_id,
 325     const StatusChangeCallback& status_change_cb,
 326     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 327     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 328     : cast_environment_(cast_environment),
 329       create_video_encode_memory_cb_(create_video_encode_memory_cb),
 330       frame_size_(frame_size),
 331       bit_rate_(video_config.start_bitrate),
 332       key_frame_requested_(false),
 333       weak_factory_(this) {
 334   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 335   DCHECK_GT(video_config.max_frame_rate, 0);
 336   DCHECK(!frame_size_.IsEmpty());
 337   DCHECK(!status_change_cb.is_null());
 338   DCHECK(!create_vea_cb.is_null());
 339   DCHECK(!create_video_encode_memory_cb_.is_null());
 340   DCHECK_GT(bit_rate_, 0);
 341
 342   create_vea_cb.Run(
 343       base::Bind(&ExternalVideoEncoder::OnCreateVideoEncodeAccelerator,
 344                  weak_factory_.GetWeakPtr(),
 345                  video_config,
 346                  first_frame_id,
 347                  status_change_cb));
 348 }
 349
 350 ExternalVideoEncoder::~ExternalVideoEncoder() {
 351 }
 352
 353 bool ExternalVideoEncoder::EncodeVideoFrame(
 354     const scoped_refptr<media::VideoFrame>& video_frame,
 355     const base::TimeTicks& reference_time,
 356     const FrameEncodedCallback& frame_encoded_callback) {
 357   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 358   DCHECK(!frame_encoded_callback.is_null());
 359
 360   if (!client_ || video_frame->visible_rect().size() != frame_size_)
 361     return false;
 362
 363   client_->task_runner()->PostTask(FROM_HERE,
 364                                    base::Bind(&VEAClientImpl::EncodeVideoFrame,
 365                                               client_,
 366                                               video_frame,
 367                                               reference_time,
 368                                               key_frame_requested_,
 369                                               frame_encoded_callback));
 370   key_frame_requested_ = false;
 371   return true;
 372 }
 373
 374 void ExternalVideoEncoder::SetBitRate(int new_bit_rate) {
 375   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 376   DCHECK_GT(new_bit_rate, 0);
 377
 378   bit_rate_ = new_bit_rate;
 379   if (!client_)
 380     return;
 381   client_->task_runner()->PostTask(
 382       FROM_HERE, base::Bind(&VEAClientImpl::SetBitRate, client_, bit_rate_));
 383 }
 384
 385 void ExternalVideoEncoder::GenerateKeyFrame() {
 386   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 387   key_frame_requested_ = true;
 388 }
 389
 390 void ExternalVideoEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
 391   // Do nothing.  Not supported.
 392 }
 393
 394 void ExternalVideoEncoder::OnCreateVideoEncodeAccelerator(
 395     const VideoSenderConfig& video_config,
 396     uint32 first_frame_id,
 397     const StatusChangeCallback& status_change_cb,
 398     scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner,
 399     scoped_ptr<media::VideoEncodeAccelerator> vea) {
 400   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 401
 402   // The callback will be invoked with null pointers in the case where the
 403   // system does not support or lacks the resources to provide GPU-accelerated
 404   // video encoding.
 405   if (!encoder_task_runner || !vea) {
 406     cast_environment_->PostTask(
 407         CastEnvironment::MAIN,
 408         FROM_HERE,
 409         base::Bind(status_change_cb, STATUS_CODEC_INIT_FAILED));
 410     return;
 411   }
 412
 413   VideoCodecProfile codec_profile;
 414   switch (video_config.codec) {
 415     case CODEC_VIDEO_VP8:
 416       codec_profile = media::VP8PROFILE_ANY;
 417       break;
 418     case CODEC_VIDEO_H264:
 419       codec_profile = media::H264PROFILE_MAIN;
 420       break;
 421     case CODEC_VIDEO_FAKE:
 422       NOTREACHED() << "Fake software video encoder cannot be external";
 423       // ...flow through to next case...
 424     default:
 425       cast_environment_->PostTask(
 426         CastEnvironment::MAIN,
 427         FROM_HERE,
 428         base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
 429       return;
 430   }
 431
 432   DCHECK(!client_);
 433   client_ = new VEAClientImpl(cast_environment_,
 434                               encoder_task_runner,
 435                               vea.Pass(),
 436                               video_config.max_frame_rate,
 437                               status_change_cb,
 438                               create_video_encode_memory_cb_);
 439   client_->task_runner()->PostTask(FROM_HERE,
 440                                    base::Bind(&VEAClientImpl::Initialize,
 441                                               client_,
 442                                               frame_size_,
 443                                               codec_profile,
 444                                               bit_rate_,
 445                                               first_frame_id));
 446 }
 447
 448 SizeAdaptableExternalVideoEncoder::SizeAdaptableExternalVideoEncoder(
 449     const scoped_refptr<CastEnvironment>& cast_environment,
 450     const VideoSenderConfig& video_config,
 451     const StatusChangeCallback& status_change_cb,
 452     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
 453     const CreateVideoEncodeMemoryCallback& create_video_encode_memory_cb)
 454     : SizeAdaptableVideoEncoderBase(cast_environment,
 455                                     video_config,
 456                                     status_change_cb),
 457       create_vea_cb_(create_vea_cb),
 458       create_video_encode_memory_cb_(create_video_encode_memory_cb) {}
 459
 460 SizeAdaptableExternalVideoEncoder::~SizeAdaptableExternalVideoEncoder() {}
 461
 462 scoped_ptr<VideoEncoder> SizeAdaptableExternalVideoEncoder::CreateEncoder() {
 463   return scoped_ptr<VideoEncoder>(new ExternalVideoEncoder(
 464       cast_environment(),
 465       video_config(),
 466       frame_size(),
 467       last_frame_id() + 1,
 468       CreateEncoderStatusChangeCallback(),
 469       create_vea_cb_,
 470       create_video_encode_memory_cb_));
 471 }
 472
 473 }  //  namespace cast
 474 }  //  namespace media