1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // This file contains an implementation of VideoDecodeAccelerator
6 // that utilizes hardware video decoders, which expose Video4Linux 2 API
7 // (http://linuxtv.org/downloads/v4l-dvb-apis/).
9 #ifndef CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_
10 #define CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_
15 #include "base/callback_forward.h"
16 #include "base/memory/linked_ptr.h"
17 #include "base/memory/ref_counted.h"
18 #include "base/memory/scoped_ptr.h"
19 #include "base/synchronization/waitable_event.h"
20 #include "base/threading/thread.h"
21 #include "content/common/content_export.h"
22 #include "content/common/gpu/media/v4l2_device.h"
23 #include "media/base/limits.h"
24 #include "media/base/video_decoder_config.h"
25 #include "media/video/picture.h"
26 #include "media/video/video_decode_accelerator.h"
27 #include "ui/gfx/geometry/size.h"
28 #include "ui/gl/gl_bindings.h"
31 class MessageLoopProxy
;
39 // This class handles video accelerators directly through a V4L2 device exported
40 // by the hardware blocks.
42 // The threading model of this class is driven by the fact that it needs to
43 // interface two fundamentally different event queues -- the one Chromium
44 // provides through MessageLoop, and the one driven by the V4L2 devices which
45 // is waited on with epoll(). There are three threads involved in this class:
47 // * The child thread, which is the main GPU process thread which calls the
48 // media::VideoDecodeAccelerator entry points. Calls from this thread
49 // generally do not block (with the exception of Initialize() and Destroy()).
50 // They post tasks to the decoder_thread_, which actually services the task
51 // and calls back when complete through the
52 // media::VideoDecodeAccelerator::Client interface.
53 // * The decoder_thread_, owned by this class. It services API tasks, through
54 // the *Task() routines, as well as V4L2 device events, through
55 // ServiceDeviceTask(). Almost all state modification is done on this thread
56 // (this doesn't include buffer (re)allocation sequence, see below).
57 // * The device_poll_thread_, owned by this class. All it does is epoll() on
58 // the V4L2 in DevicePollTask() and schedule a ServiceDeviceTask() on the
59 // decoder_thread_ when something interesting happens.
60 // TODO(sheu): replace this thread with an TYPE_IO decoder_thread_.
62 // Note that this class has (almost) no locks, apart from the pictures_assigned_
63 // WaitableEvent. Everything (apart from buffer (re)allocation) is serviced on
64 // the decoder_thread_, so there are no synchronization issues.
65 // ... well, there are, but it's a matter of getting messages posted in the
66 // right order, not fiddling with locks.
67 // Buffer creation is a two-step process that is serviced partially on the
68 // Child thread, because we need to wait for the client to provide textures
69 // for the buffers we allocate. We cannot keep the decoder thread running while
70 // the client allocates Pictures for us, because we need to REQBUFS first to get
71 // the required number of output buffers from the device and that cannot be done
72 // unless we free the previous set of buffers, leaving the decoding in a
73 // inoperable state for the duration of the wait for Pictures. So to prevent
74 // subtle races (esp. if we get Reset() in the meantime), we block the decoder
75 // thread while we wait for AssignPictureBuffers from the client.
76 class CONTENT_EXPORT V4L2VideoDecodeAccelerator
77 : public media::VideoDecodeAccelerator
{
79 V4L2VideoDecodeAccelerator(
80 EGLDisplay egl_display
,
81 EGLContext egl_context
,
82 const base::WeakPtr
<Client
>& io_client_
,
83 const base::Callback
<bool(void)>& make_context_current
,
84 const scoped_refptr
<V4L2Device
>& device
,
85 const scoped_refptr
<base::MessageLoopProxy
>& io_message_loop_proxy
);
86 ~V4L2VideoDecodeAccelerator() override
;
88 // media::VideoDecodeAccelerator implementation.
89 // Note: Initialize() and Destroy() are synchronous.
90 bool Initialize(media::VideoCodecProfile profile
,
91 Client
* client
) override
;
92 void Decode(const media::BitstreamBuffer
& bitstream_buffer
) override
;
93 void AssignPictureBuffers(
94 const std::vector
<media::PictureBuffer
>& buffers
) override
;
95 void ReusePictureBuffer(int32 picture_buffer_id
) override
;
96 void Flush() override
;
97 void Reset() override
;
98 void Destroy() override
;
99 bool CanDecodeOnIOThread() override
;
101 static media::VideoDecodeAccelerator::SupportedProfiles
102 GetSupportedProfiles();
105 // These are rather subjectively tuned.
107 kInputBufferCount
= 8,
108 // TODO(posciak): determine input buffer size based on level limits.
109 // See http://crbug.com/255116.
110 // Input bitstream buffer size for up to 1080p streams.
111 kInputBufferMaxSizeFor1080p
= 1024 * 1024,
112 // Input bitstream buffer size for up to 4k streams.
113 kInputBufferMaxSizeFor4k
= 4 * kInputBufferMaxSizeFor1080p
,
114 // Number of output buffers to use for each VDA stage above what's required
115 // by the decoder (e.g. DPB size, in H264). We need
116 // media::limits::kMaxVideoFrames to fill up the GpuVideoDecode pipeline,
117 // and +1 for a frame in transit.
118 kDpbOutputBufferExtraCount
= media::limits::kMaxVideoFrames
+ 1,
121 // Internal state of the decoder.
123 kUninitialized
, // Initialize() not yet called.
124 kInitialized
, // Initialize() returned true; ready to start decoding.
125 kDecoding
, // DecodeBufferInitial() successful; decoding frames.
126 kResetting
, // Presently resetting.
127 kAfterReset
, // After Reset(), ready to start decoding again.
128 kChangingResolution
, // Performing resolution change, all remaining
129 // pre-change frames decoded and processed.
130 kError
, // Error in kDecoding state.
134 kFlushBufferId
= -2 // Buffer id for flush buffer, queued by FlushTask().
137 // Auto-destruction reference for BitstreamBuffer, for message-passing from
138 // Decode() to DecodeTask().
139 struct BitstreamBufferRef
;
141 // Auto-destruction reference for EGLSync (for message-passing).
142 struct EGLSyncKHRRef
;
144 // Record for decoded pictures that can be sent to PictureReady.
145 struct PictureRecord
;
147 // Record for input buffers.
151 bool at_device
; // held by device.
152 void* address
; // mmap() address.
153 size_t length
; // mmap() length.
154 off_t bytes_used
; // bytes filled in the mmap() segment.
155 int32 input_id
; // triggering input_id as given to Decode().
158 // Record for output buffers.
159 struct OutputRecord
{
162 bool at_device
; // held by device.
163 bool at_client
; // held by client.
164 EGLImageKHR egl_image
; // EGLImageKHR for the output buffer.
165 EGLSyncKHR egl_sync
; // sync the compositor's use of the EGLImage.
166 int32 picture_id
; // picture buffer id as returned to PictureReady().
167 bool cleared
; // Whether the texture is cleared and safe to render
168 // from. See TextureManager for details.
172 // Decoding tasks, to be run on decode_thread_.
175 // Enqueue a BitstreamBuffer to decode. This will enqueue a buffer to the
176 // decoder_input_queue_, then queue a DecodeBufferTask() to actually decode
178 void DecodeTask(const media::BitstreamBuffer
& bitstream_buffer
);
180 // Decode from the buffers queued in decoder_input_queue_. Calls
181 // DecodeBufferInitial() or DecodeBufferContinue() as appropriate.
182 void DecodeBufferTask();
183 // Advance to the next fragment that begins a frame.
184 bool AdvanceFrameFragment(const uint8
* data
, size_t size
, size_t* endpos
);
185 // Schedule another DecodeBufferTask() if we're behind.
186 void ScheduleDecodeBufferTaskIfNeeded();
188 // Return true if we should continue to schedule DecodeBufferTask()s after
189 // completion. Store the amount of input actually consumed in |endpos|.
190 bool DecodeBufferInitial(const void* data
, size_t size
, size_t* endpos
);
191 bool DecodeBufferContinue(const void* data
, size_t size
);
193 // Accumulate data for the next frame to decode. May return false in
194 // non-error conditions; for example when pipeline is full and should be
196 bool AppendToInputFrame(const void* data
, size_t size
);
197 // Flush data for one decoded frame.
198 bool FlushInputFrame();
200 // Service I/O on the V4L2 devices. This task should only be scheduled from
201 // DevicePollTask(). If |event_pending| is true, one or more events
202 // on file descriptor are pending.
203 void ServiceDeviceTask(bool event_pending
);
204 // Handle the various device queues.
207 // Handle incoming events.
208 void DequeueEvents();
209 // Enqueue a buffer on the corresponding queue.
210 bool EnqueueInputRecord();
211 bool EnqueueOutputRecord();
213 // Process a ReusePictureBuffer() API call. The API call create an EGLSync
214 // object on the main (GPU process) thread; we will record this object so we
215 // can wait on it before reusing the buffer.
216 void ReusePictureBufferTask(int32 picture_buffer_id
,
217 scoped_ptr
<EGLSyncKHRRef
> egl_sync_ref
);
219 // Flush() task. Child thread should not submit any more buffers until it
220 // receives the NotifyFlushDone callback. This task will schedule an empty
221 // BitstreamBufferRef (with input_id == kFlushBufferId) to perform the flush.
223 // Notify the client of a flush completion, if required. This should be
224 // called any time a relevant queue could potentially be emptied: see
225 // function definition.
226 void NotifyFlushDoneIfNeeded();
228 // Reset() task. This task will schedule a ResetDoneTask() that will send
229 // the NotifyResetDone callback, then set the decoder state to kResetting so
230 // that all intervening tasks will drain.
232 // ResetDoneTask() will set the decoder state back to kAfterReset, so
233 // subsequent decoding can continue.
234 void ResetDoneTask();
236 // Device destruction task.
239 // Attempt to start/stop device_poll_thread_.
240 bool StartDevicePoll();
241 // If |keep_input_state| is true, don't reset input state; used during
242 // resolution change.
243 bool StopDevicePoll(bool keep_input_state
);
245 void StartResolutionChangeIfNeeded();
246 void FinishResolutionChange();
248 // Try to get output format and visible size, detected after parsing the
249 // beginning of the stream. Sets |again| to true if more parsing is needed.
250 // |visible_size| could be nullptr and ignored.
251 bool GetFormatInfo(struct v4l2_format
* format
,
252 gfx::Size
* visible_size
,
254 // Create output buffers for the given |format| and |visible_size|.
255 bool CreateBuffersForFormat(const struct v4l2_format
& format
,
256 const gfx::Size
& visible_size
);
258 // Try to get |visible_size|. Return visible size, or, if querying it is not
259 // supported or produces invalid size, return |coded_size| instead.
260 gfx::Size
GetVisibleSize(const gfx::Size
& coded_size
);
263 // Device tasks, to be run on device_poll_thread_.
267 void DevicePollTask(bool poll_device
);
270 // Safe from any thread.
273 // Error notification (using PostTask() to child thread, if necessary).
274 void NotifyError(Error error
);
276 // Set the decoder_state_ to kError and notify the client (if necessary).
277 void SetErrorState(Error error
);
280 // Other utility functions. Called on decoder_thread_, unless
281 // decoder_thread_ is not yet started, in which case the child thread can call
282 // these (e.g. in Initialize() or Destroy()).
285 // Create the buffers we need.
286 bool CreateInputBuffers();
287 bool CreateOutputBuffers();
289 // Set input and output formats before starting decode.
293 // Methods run on child thread.
297 void DestroyInputBuffers();
298 // In contrast to DestroyInputBuffers, which is called only from destructor,
299 // we call DestroyOutputBuffers also during playback, on resolution change.
300 // Even if anything fails along the way, we still want to go on and clean
301 // up as much as possible, so return false if this happens, so that the
302 // caller can error out on resolution change.
303 bool DestroyOutputBuffers();
304 void ResolutionChangeDestroyBuffers();
306 // Send decoded pictures to PictureReady.
307 void SendPictureReady();
309 // Callback that indicates a picture has been cleared.
310 void PictureCleared();
312 // This method determines whether a resolution change event processing
313 // is indeed required by returning true iff:
314 // - width or height of the new format is different than previous format; or
315 // - V4L2_CID_MIN_BUFFERS_FOR_CAPTURE has changed.
316 bool IsResolutionChangeNecessary();
318 // Our original calling message loop for the child thread.
319 scoped_refptr
<base::MessageLoopProxy
> child_message_loop_proxy_
;
321 // Message loop of the IO thread.
322 scoped_refptr
<base::MessageLoopProxy
> io_message_loop_proxy_
;
324 // WeakPtr<> pointing to |this| for use in posting tasks from the decoder or
325 // device worker threads back to the child thread. Because the worker threads
326 // are members of this class, any task running on those threads is guaranteed
327 // that this object is still alive. As a result, tasks posted from the child
328 // thread to the decoder or device thread should use base::Unretained(this),
329 // and tasks posted the other way should use |weak_this_|.
330 base::WeakPtr
<V4L2VideoDecodeAccelerator
> weak_this_
;
332 // To expose client callbacks from VideoDecodeAccelerator.
333 // NOTE: all calls to these objects *MUST* be executed on
334 // child_message_loop_proxy_.
335 scoped_ptr
<base::WeakPtrFactory
<Client
> > client_ptr_factory_
;
336 base::WeakPtr
<Client
> client_
;
337 // Callbacks to |io_client_| must be executed on |io_message_loop_proxy_|.
338 base::WeakPtr
<Client
> io_client_
;
341 // Decoder state, owned and operated by decoder_thread_.
342 // Before decoder_thread_ has started, the decoder state is managed by
343 // the child (main) thread. After decoder_thread_ has started, the decoder
344 // thread should be the only one managing these.
347 // This thread services tasks posted from the VDA API entry points by the
348 // child thread and device service callbacks posted from the device thread.
349 base::Thread decoder_thread_
;
350 // Decoder state machine state.
351 State decoder_state_
;
352 // BitstreamBuffer we're presently reading.
353 scoped_ptr
<BitstreamBufferRef
> decoder_current_bitstream_buffer_
;
354 // The V4L2Device this class is operating upon.
355 scoped_refptr
<V4L2Device
> device_
;
356 // FlushTask() and ResetTask() should not affect buffers that have been
357 // queued afterwards. For flushing or resetting the pipeline then, we will
358 // delay these buffers until after the flush or reset completes.
359 int decoder_delay_bitstream_buffer_id_
;
360 // Input buffer we're presently filling.
361 int decoder_current_input_buffer_
;
362 // We track the number of buffer decode tasks we have scheduled, since each
363 // task execution should complete one buffer. If we fall behind (due to
364 // resource backpressure, etc.), we'll have to schedule more to catch up.
365 int decoder_decode_buffer_tasks_scheduled_
;
366 // Picture buffers held by the client.
367 int decoder_frames_at_client_
;
369 bool decoder_flushing_
;
370 // Got a notification from driver that it reached resolution change point
372 bool resolution_change_pending_
;
373 // Got a reset request while we were performing resolution change.
374 bool resolution_change_reset_pending_
;
375 // Input queue for decoder_thread_: BitstreamBuffers in.
376 std::queue
<linked_ptr
<BitstreamBufferRef
> > decoder_input_queue_
;
377 // For H264 decode, hardware requires that we send it frame-sized chunks.
378 // We'll need to parse the stream.
379 scoped_ptr
<media::H264Parser
> decoder_h264_parser_
;
380 // Set if the decoder has a pending incomplete frame in an input buffer.
381 bool decoder_partial_frame_pending_
;
384 // Hardware state and associated queues. Since decoder_thread_ services
385 // the hardware, decoder_thread_ owns these too.
386 // output_buffer_map_, free_output_buffers_ and output_planes_count_ are an
387 // exception during the buffer (re)allocation sequence, when the
388 // decoder_thread_ is blocked briefly while the Child thread manipulates
392 // Completed decode buffers.
393 std::queue
<int> input_ready_queue_
;
395 // Input buffer state.
396 bool input_streamon_
;
397 // Input buffers enqueued to device.
398 int input_buffer_queued_count_
;
399 // Input buffers ready to use, as a LIFO since we don't care about ordering.
400 std::vector
<int> free_input_buffers_
;
401 // Mapping of int index to input buffer record.
402 std::vector
<InputRecord
> input_buffer_map_
;
404 // Output buffer state.
405 bool output_streamon_
;
406 // Output buffers enqueued to device.
407 int output_buffer_queued_count_
;
408 // Output buffers ready to use, as a FIFO since we want oldest-first to hide
409 // synchronization latency with GL.
410 std::queue
<int> free_output_buffers_
;
411 // Mapping of int index to output buffer record.
412 std::vector
<OutputRecord
> output_buffer_map_
;
413 // Required size of DPB for decoding.
414 int output_dpb_size_
;
416 // Number of planes (i.e. separate memory buffers) for output.
417 size_t output_planes_count_
;
419 // Pictures that are ready but not sent to PictureReady yet.
420 std::queue
<PictureRecord
> pending_picture_ready_
;
422 // The number of pictures that are sent to PictureReady and will be cleared.
423 int picture_clearing_count_
;
425 // Used by the decoder thread to wait for AssignPictureBuffers to arrive
426 // to avoid races with potential Reset requests.
427 base::WaitableEvent pictures_assigned_
;
429 // Output picture coded size.
430 gfx::Size coded_size_
;
432 // Output picture visible size.
433 gfx::Size visible_size_
;
436 // The device polling thread handles notifications of V4L2 device changes.
440 base::Thread device_poll_thread_
;
443 // Other state, held by the child (main) thread.
446 // Make our context current before running any EGL entry points.
447 base::Callback
<bool(void)> make_context_current_
;
450 EGLDisplay egl_display_
;
451 EGLContext egl_context_
;
453 // The codec we'll be decoding for.
454 media::VideoCodecProfile video_profile_
;
455 // Chosen output format.
456 uint32_t output_format_fourcc_
;
458 // The WeakPtrFactory for |weak_this_|.
459 base::WeakPtrFactory
<V4L2VideoDecodeAccelerator
> weak_this_factory_
;
461 DISALLOW_COPY_AND_ASSIGN(V4L2VideoDecodeAccelerator
);
464 } // namespace content
466 #endif // CONTENT_COMMON_GPU_MEDIA_V4L2_VIDEO_DECODE_ACCELERATOR_H_