2 * Copyright 2013 Advanced Micro Devices, Inc.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
25 * Authors: Christian König <christian.koenig@amd.com>
28 #include <linux/firmware.h>
29 #include <linux/module.h>
34 #include "amdgpu_pm.h"
35 #include "amdgpu_vce.h"
38 /* 1 second timeout */
39 #define VCE_IDLE_TIMEOUT msecs_to_jiffies(1000)
42 #ifdef CONFIG_DRM_AMDGPU_CIK
43 #define FIRMWARE_BONAIRE "amdgpu/bonaire_vce.bin"
44 #define FIRMWARE_KABINI "amdgpu/kabini_vce.bin"
45 #define FIRMWARE_KAVERI "amdgpu/kaveri_vce.bin"
46 #define FIRMWARE_HAWAII "amdgpu/hawaii_vce.bin"
47 #define FIRMWARE_MULLINS "amdgpu/mullins_vce.bin"
49 #define FIRMWARE_TONGA "amdgpu/tonga_vce.bin"
50 #define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin"
51 #define FIRMWARE_FIJI "amdgpu/fiji_vce.bin"
52 #define FIRMWARE_STONEY "amdgpu/stoney_vce.bin"
53 #define FIRMWARE_POLARIS10 "amdgpu/polaris10_vce.bin"
54 #define FIRMWARE_POLARIS11 "amdgpu/polaris11_vce.bin"
55 #define FIRMWARE_POLARIS12 "amdgpu/polaris12_vce.bin"
56 #define FIRMWARE_VEGAM "amdgpu/vegam_vce.bin"
58 #define FIRMWARE_VEGA10 "amdgpu/vega10_vce.bin"
59 #define FIRMWARE_VEGA12 "amdgpu/vega12_vce.bin"
60 #define FIRMWARE_VEGA20 "amdgpu/vega20_vce.bin"
62 #ifdef CONFIG_DRM_AMDGPU_CIK
63 MODULE_FIRMWARE(FIRMWARE_BONAIRE
);
64 MODULE_FIRMWARE(FIRMWARE_KABINI
);
65 MODULE_FIRMWARE(FIRMWARE_KAVERI
);
66 MODULE_FIRMWARE(FIRMWARE_HAWAII
);
67 MODULE_FIRMWARE(FIRMWARE_MULLINS
);
69 MODULE_FIRMWARE(FIRMWARE_TONGA
);
70 MODULE_FIRMWARE(FIRMWARE_CARRIZO
);
71 MODULE_FIRMWARE(FIRMWARE_FIJI
);
72 MODULE_FIRMWARE(FIRMWARE_STONEY
);
73 MODULE_FIRMWARE(FIRMWARE_POLARIS10
);
74 MODULE_FIRMWARE(FIRMWARE_POLARIS11
);
75 MODULE_FIRMWARE(FIRMWARE_POLARIS12
);
76 MODULE_FIRMWARE(FIRMWARE_VEGAM
);
78 MODULE_FIRMWARE(FIRMWARE_VEGA10
);
79 MODULE_FIRMWARE(FIRMWARE_VEGA12
);
80 MODULE_FIRMWARE(FIRMWARE_VEGA20
);
82 static void amdgpu_vce_idle_work_handler(struct work_struct
*work
);
83 static int amdgpu_vce_get_create_msg(struct amdgpu_ring
*ring
, uint32_t handle
,
85 struct dma_fence
**fence
);
86 static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring
*ring
, uint32_t handle
,
87 bool direct
, struct dma_fence
**fence
);
90 * amdgpu_vce_init - allocate memory, load vce firmware
92 * @adev: amdgpu_device pointer
94 * First step to get VCE online, allocate memory and load the firmware
96 int amdgpu_vce_sw_init(struct amdgpu_device
*adev
, unsigned long size
)
99 const struct common_firmware_header
*hdr
;
100 unsigned ucode_version
, version_major
, version_minor
, binary_id
;
103 switch (adev
->asic_type
) {
104 #ifdef CONFIG_DRM_AMDGPU_CIK
106 fw_name
= FIRMWARE_BONAIRE
;
109 fw_name
= FIRMWARE_KAVERI
;
112 fw_name
= FIRMWARE_KABINI
;
115 fw_name
= FIRMWARE_HAWAII
;
118 fw_name
= FIRMWARE_MULLINS
;
122 fw_name
= FIRMWARE_TONGA
;
125 fw_name
= FIRMWARE_CARRIZO
;
128 fw_name
= FIRMWARE_FIJI
;
131 fw_name
= FIRMWARE_STONEY
;
134 fw_name
= FIRMWARE_POLARIS10
;
137 fw_name
= FIRMWARE_POLARIS11
;
140 fw_name
= FIRMWARE_POLARIS12
;
143 fw_name
= FIRMWARE_VEGAM
;
146 fw_name
= FIRMWARE_VEGA10
;
149 fw_name
= FIRMWARE_VEGA12
;
152 fw_name
= FIRMWARE_VEGA20
;
159 r
= request_firmware(&adev
->vce
.fw
, fw_name
, adev
->dev
);
161 dev_err(adev
->dev
, "amdgpu_vce: Can't load firmware \"%s\"\n",
166 r
= amdgpu_ucode_validate(adev
->vce
.fw
);
168 dev_err(adev
->dev
, "amdgpu_vce: Can't validate firmware \"%s\"\n",
170 release_firmware(adev
->vce
.fw
);
175 hdr
= (const struct common_firmware_header
*)adev
->vce
.fw
->data
;
177 ucode_version
= le32_to_cpu(hdr
->ucode_version
);
178 version_major
= (ucode_version
>> 20) & 0xfff;
179 version_minor
= (ucode_version
>> 8) & 0xfff;
180 binary_id
= ucode_version
& 0xff;
181 DRM_INFO("Found VCE firmware Version: %hhd.%hhd Binary ID: %hhd\n",
182 version_major
, version_minor
, binary_id
);
183 adev
->vce
.fw_version
= ((version_major
<< 24) | (version_minor
<< 16) |
186 r
= amdgpu_bo_create_kernel(adev
, size
, PAGE_SIZE
,
187 AMDGPU_GEM_DOMAIN_VRAM
, &adev
->vce
.vcpu_bo
,
188 &adev
->vce
.gpu_addr
, &adev
->vce
.cpu_addr
);
190 dev_err(adev
->dev
, "(%d) failed to allocate VCE bo\n", r
);
194 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
) {
195 atomic_set(&adev
->vce
.handles
[i
], 0);
196 adev
->vce
.filp
[i
] = NULL
;
199 INIT_DELAYED_WORK(&adev
->vce
.idle_work
, amdgpu_vce_idle_work_handler
);
200 mutex_init(&adev
->vce
.idle_mutex
);
206 * amdgpu_vce_fini - free memory
208 * @adev: amdgpu_device pointer
210 * Last step on VCE teardown, free firmware memory
212 int amdgpu_vce_sw_fini(struct amdgpu_device
*adev
)
216 if (adev
->vce
.vcpu_bo
== NULL
)
219 cancel_delayed_work_sync(&adev
->vce
.idle_work
);
220 drm_sched_entity_destroy(&adev
->vce
.entity
);
222 amdgpu_bo_free_kernel(&adev
->vce
.vcpu_bo
, &adev
->vce
.gpu_addr
,
223 (void **)&adev
->vce
.cpu_addr
);
225 for (i
= 0; i
< adev
->vce
.num_rings
; i
++)
226 amdgpu_ring_fini(&adev
->vce
.ring
[i
]);
228 release_firmware(adev
->vce
.fw
);
229 mutex_destroy(&adev
->vce
.idle_mutex
);
235 * amdgpu_vce_entity_init - init entity
237 * @adev: amdgpu_device pointer
240 int amdgpu_vce_entity_init(struct amdgpu_device
*adev
)
242 struct amdgpu_ring
*ring
;
243 struct drm_gpu_scheduler
*sched
;
246 ring
= &adev
->vce
.ring
[0];
247 sched
= &ring
->sched
;
248 r
= drm_sched_entity_init(&adev
->vce
.entity
, DRM_SCHED_PRIORITY_NORMAL
,
251 DRM_ERROR("Failed setting up VCE run queue.\n");
259 * amdgpu_vce_suspend - unpin VCE fw memory
261 * @adev: amdgpu_device pointer
264 int amdgpu_vce_suspend(struct amdgpu_device
*adev
)
268 cancel_delayed_work_sync(&adev
->vce
.idle_work
);
270 if (adev
->vce
.vcpu_bo
== NULL
)
273 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
)
274 if (atomic_read(&adev
->vce
.handles
[i
]))
277 if (i
== AMDGPU_MAX_VCE_HANDLES
)
280 /* TODO: suspending running encoding sessions isn't supported */
285 * amdgpu_vce_resume - pin VCE fw memory
287 * @adev: amdgpu_device pointer
290 int amdgpu_vce_resume(struct amdgpu_device
*adev
)
293 const struct common_firmware_header
*hdr
;
297 if (adev
->vce
.vcpu_bo
== NULL
)
300 r
= amdgpu_bo_reserve(adev
->vce
.vcpu_bo
, false);
302 dev_err(adev
->dev
, "(%d) failed to reserve VCE bo\n", r
);
306 r
= amdgpu_bo_kmap(adev
->vce
.vcpu_bo
, &cpu_addr
);
308 amdgpu_bo_unreserve(adev
->vce
.vcpu_bo
);
309 dev_err(adev
->dev
, "(%d) VCE map failed\n", r
);
313 hdr
= (const struct common_firmware_header
*)adev
->vce
.fw
->data
;
314 offset
= le32_to_cpu(hdr
->ucode_array_offset_bytes
);
315 memcpy_toio(cpu_addr
, adev
->vce
.fw
->data
+ offset
,
316 adev
->vce
.fw
->size
- offset
);
318 amdgpu_bo_kunmap(adev
->vce
.vcpu_bo
);
320 amdgpu_bo_unreserve(adev
->vce
.vcpu_bo
);
326 * amdgpu_vce_idle_work_handler - power off VCE
328 * @work: pointer to work structure
330 * power of VCE when it's not used any more
332 static void amdgpu_vce_idle_work_handler(struct work_struct
*work
)
334 struct amdgpu_device
*adev
=
335 container_of(work
, struct amdgpu_device
, vce
.idle_work
.work
);
336 unsigned i
, count
= 0;
338 for (i
= 0; i
< adev
->vce
.num_rings
; i
++)
339 count
+= amdgpu_fence_count_emitted(&adev
->vce
.ring
[i
]);
342 if (adev
->pm
.dpm_enabled
) {
343 amdgpu_dpm_enable_vce(adev
, false);
345 amdgpu_asic_set_vce_clocks(adev
, 0, 0);
346 amdgpu_device_ip_set_powergating_state(adev
, AMD_IP_BLOCK_TYPE_VCE
,
348 amdgpu_device_ip_set_clockgating_state(adev
, AMD_IP_BLOCK_TYPE_VCE
,
352 schedule_delayed_work(&adev
->vce
.idle_work
, VCE_IDLE_TIMEOUT
);
357 * amdgpu_vce_ring_begin_use - power up VCE
361 * Make sure VCE is powerd up when we want to use it
363 void amdgpu_vce_ring_begin_use(struct amdgpu_ring
*ring
)
365 struct amdgpu_device
*adev
= ring
->adev
;
368 if (amdgpu_sriov_vf(adev
))
371 mutex_lock(&adev
->vce
.idle_mutex
);
372 set_clocks
= !cancel_delayed_work_sync(&adev
->vce
.idle_work
);
374 if (adev
->pm
.dpm_enabled
) {
375 amdgpu_dpm_enable_vce(adev
, true);
377 amdgpu_asic_set_vce_clocks(adev
, 53300, 40000);
378 amdgpu_device_ip_set_clockgating_state(adev
, AMD_IP_BLOCK_TYPE_VCE
,
379 AMD_CG_STATE_UNGATE
);
380 amdgpu_device_ip_set_powergating_state(adev
, AMD_IP_BLOCK_TYPE_VCE
,
381 AMD_PG_STATE_UNGATE
);
385 mutex_unlock(&adev
->vce
.idle_mutex
);
389 * amdgpu_vce_ring_end_use - power VCE down
393 * Schedule work to power VCE down again
395 void amdgpu_vce_ring_end_use(struct amdgpu_ring
*ring
)
397 if (!amdgpu_sriov_vf(ring
->adev
))
398 schedule_delayed_work(&ring
->adev
->vce
.idle_work
, VCE_IDLE_TIMEOUT
);
402 * amdgpu_vce_free_handles - free still open VCE handles
404 * @adev: amdgpu_device pointer
405 * @filp: drm file pointer
407 * Close all VCE handles still open by this file pointer
409 void amdgpu_vce_free_handles(struct amdgpu_device
*adev
, struct drm_file
*filp
)
411 struct amdgpu_ring
*ring
= &adev
->vce
.ring
[0];
413 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
) {
414 uint32_t handle
= atomic_read(&adev
->vce
.handles
[i
]);
416 if (!handle
|| adev
->vce
.filp
[i
] != filp
)
419 r
= amdgpu_vce_get_destroy_msg(ring
, handle
, false, NULL
);
421 DRM_ERROR("Error destroying VCE handle (%d)!\n", r
);
423 adev
->vce
.filp
[i
] = NULL
;
424 atomic_set(&adev
->vce
.handles
[i
], 0);
429 * amdgpu_vce_get_create_msg - generate a VCE create msg
431 * @adev: amdgpu_device pointer
432 * @ring: ring we should submit the msg to
433 * @handle: VCE session handle to use
434 * @fence: optional fence to return
436 * Open up a stream for HW test
438 static int amdgpu_vce_get_create_msg(struct amdgpu_ring
*ring
, uint32_t handle
,
439 struct amdgpu_bo
*bo
,
440 struct dma_fence
**fence
)
442 const unsigned ib_size_dw
= 1024;
443 struct amdgpu_job
*job
;
444 struct amdgpu_ib
*ib
;
445 struct dma_fence
*f
= NULL
;
449 r
= amdgpu_job_alloc_with_ib(ring
->adev
, ib_size_dw
* 4, &job
);
455 addr
= amdgpu_bo_gpu_offset(bo
);
457 /* stitch together an VCE create msg */
459 ib
->ptr
[ib
->length_dw
++] = 0x0000000c; /* len */
460 ib
->ptr
[ib
->length_dw
++] = 0x00000001; /* session cmd */
461 ib
->ptr
[ib
->length_dw
++] = handle
;
463 if ((ring
->adev
->vce
.fw_version
>> 24) >= 52)
464 ib
->ptr
[ib
->length_dw
++] = 0x00000040; /* len */
466 ib
->ptr
[ib
->length_dw
++] = 0x00000030; /* len */
467 ib
->ptr
[ib
->length_dw
++] = 0x01000001; /* create cmd */
468 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
469 ib
->ptr
[ib
->length_dw
++] = 0x00000042;
470 ib
->ptr
[ib
->length_dw
++] = 0x0000000a;
471 ib
->ptr
[ib
->length_dw
++] = 0x00000001;
472 ib
->ptr
[ib
->length_dw
++] = 0x00000080;
473 ib
->ptr
[ib
->length_dw
++] = 0x00000060;
474 ib
->ptr
[ib
->length_dw
++] = 0x00000100;
475 ib
->ptr
[ib
->length_dw
++] = 0x00000100;
476 ib
->ptr
[ib
->length_dw
++] = 0x0000000c;
477 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
478 if ((ring
->adev
->vce
.fw_version
>> 24) >= 52) {
479 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
480 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
481 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
482 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
485 ib
->ptr
[ib
->length_dw
++] = 0x00000014; /* len */
486 ib
->ptr
[ib
->length_dw
++] = 0x05000005; /* feedback buffer */
487 ib
->ptr
[ib
->length_dw
++] = upper_32_bits(addr
);
488 ib
->ptr
[ib
->length_dw
++] = addr
;
489 ib
->ptr
[ib
->length_dw
++] = 0x00000001;
491 for (i
= ib
->length_dw
; i
< ib_size_dw
; ++i
)
494 r
= amdgpu_job_submit_direct(job
, ring
, &f
);
499 *fence
= dma_fence_get(f
);
504 amdgpu_job_free(job
);
509 * amdgpu_vce_get_destroy_msg - generate a VCE destroy msg
511 * @adev: amdgpu_device pointer
512 * @ring: ring we should submit the msg to
513 * @handle: VCE session handle to use
514 * @fence: optional fence to return
516 * Close up a stream for HW test or if userspace failed to do so
518 static int amdgpu_vce_get_destroy_msg(struct amdgpu_ring
*ring
, uint32_t handle
,
519 bool direct
, struct dma_fence
**fence
)
521 const unsigned ib_size_dw
= 1024;
522 struct amdgpu_job
*job
;
523 struct amdgpu_ib
*ib
;
524 struct dma_fence
*f
= NULL
;
527 r
= amdgpu_job_alloc_with_ib(ring
->adev
, ib_size_dw
* 4, &job
);
533 /* stitch together an VCE destroy msg */
535 ib
->ptr
[ib
->length_dw
++] = 0x0000000c; /* len */
536 ib
->ptr
[ib
->length_dw
++] = 0x00000001; /* session cmd */
537 ib
->ptr
[ib
->length_dw
++] = handle
;
539 ib
->ptr
[ib
->length_dw
++] = 0x00000020; /* len */
540 ib
->ptr
[ib
->length_dw
++] = 0x00000002; /* task info */
541 ib
->ptr
[ib
->length_dw
++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
542 ib
->ptr
[ib
->length_dw
++] = 0x00000001; /* destroy session */
543 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
544 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
545 ib
->ptr
[ib
->length_dw
++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
546 ib
->ptr
[ib
->length_dw
++] = 0x00000000;
548 ib
->ptr
[ib
->length_dw
++] = 0x00000008; /* len */
549 ib
->ptr
[ib
->length_dw
++] = 0x02000001; /* destroy cmd */
551 for (i
= ib
->length_dw
; i
< ib_size_dw
; ++i
)
555 r
= amdgpu_job_submit_direct(job
, ring
, &f
);
557 r
= amdgpu_job_submit(job
, &ring
->adev
->vce
.entity
,
558 AMDGPU_FENCE_OWNER_UNDEFINED
, &f
);
563 *fence
= dma_fence_get(f
);
568 amdgpu_job_free(job
);
573 * amdgpu_vce_cs_validate_bo - make sure not to cross 4GB boundary
576 * @lo: address of lower dword
577 * @hi: address of higher dword
578 * @size: minimum size
579 * @index: bs/fb index
581 * Make sure that no BO cross a 4GB boundary.
583 static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
,
584 int lo
, int hi
, unsigned size
, int32_t index
)
586 int64_t offset
= ((uint64_t)size
) * ((int64_t)index
);
587 struct ttm_operation_ctx ctx
= { false, false };
588 struct amdgpu_bo_va_mapping
*mapping
;
589 unsigned i
, fpfn
, lpfn
;
590 struct amdgpu_bo
*bo
;
594 addr
= ((uint64_t)amdgpu_get_ib_value(p
, ib_idx
, lo
)) |
595 ((uint64_t)amdgpu_get_ib_value(p
, ib_idx
, hi
)) << 32;
598 fpfn
= PAGE_ALIGN(offset
) >> PAGE_SHIFT
;
599 lpfn
= 0x100000000ULL
>> PAGE_SHIFT
;
602 lpfn
= (0x100000000ULL
- PAGE_ALIGN(offset
)) >> PAGE_SHIFT
;
605 r
= amdgpu_cs_find_mapping(p
, addr
, &bo
, &mapping
);
607 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
608 addr
, lo
, hi
, size
, index
);
612 for (i
= 0; i
< bo
->placement
.num_placement
; ++i
) {
613 bo
->placements
[i
].fpfn
= max(bo
->placements
[i
].fpfn
, fpfn
);
614 bo
->placements
[i
].lpfn
= bo
->placements
[i
].lpfn
?
615 min(bo
->placements
[i
].lpfn
, lpfn
) : lpfn
;
617 return ttm_bo_validate(&bo
->tbo
, &bo
->placement
, &ctx
);
622 * amdgpu_vce_cs_reloc - command submission relocation
625 * @lo: address of lower dword
626 * @hi: address of higher dword
627 * @size: minimum size
629 * Patch relocation inside command stream with real buffer address
631 static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
,
632 int lo
, int hi
, unsigned size
, uint32_t index
)
634 struct amdgpu_bo_va_mapping
*mapping
;
635 struct amdgpu_bo
*bo
;
639 if (index
== 0xffffffff)
642 addr
= ((uint64_t)amdgpu_get_ib_value(p
, ib_idx
, lo
)) |
643 ((uint64_t)amdgpu_get_ib_value(p
, ib_idx
, hi
)) << 32;
644 addr
+= ((uint64_t)size
) * ((uint64_t)index
);
646 r
= amdgpu_cs_find_mapping(p
, addr
, &bo
, &mapping
);
648 DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
649 addr
, lo
, hi
, size
, index
);
653 if ((addr
+ (uint64_t)size
) >
654 (mapping
->last
+ 1) * AMDGPU_GPU_PAGE_SIZE
) {
655 DRM_ERROR("BO to small for addr 0x%010Lx %d %d\n",
660 addr
-= mapping
->start
* AMDGPU_GPU_PAGE_SIZE
;
661 addr
+= amdgpu_bo_gpu_offset(bo
);
662 addr
-= ((uint64_t)size
) * ((uint64_t)index
);
664 amdgpu_set_ib_value(p
, ib_idx
, lo
, lower_32_bits(addr
));
665 amdgpu_set_ib_value(p
, ib_idx
, hi
, upper_32_bits(addr
));
671 * amdgpu_vce_validate_handle - validate stream handle
674 * @handle: handle to validate
675 * @allocated: allocated a new handle?
677 * Validates the handle and return the found session index or -EINVAL
678 * we we don't have another free session index.
680 static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser
*p
,
681 uint32_t handle
, uint32_t *allocated
)
685 /* validate the handle */
686 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
) {
687 if (atomic_read(&p
->adev
->vce
.handles
[i
]) == handle
) {
688 if (p
->adev
->vce
.filp
[i
] != p
->filp
) {
689 DRM_ERROR("VCE handle collision detected!\n");
696 /* handle not found try to alloc a new one */
697 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
) {
698 if (!atomic_cmpxchg(&p
->adev
->vce
.handles
[i
], 0, handle
)) {
699 p
->adev
->vce
.filp
[i
] = p
->filp
;
700 p
->adev
->vce
.img_size
[i
] = 0;
701 *allocated
|= 1 << i
;
706 DRM_ERROR("No more free VCE handles!\n");
711 * amdgpu_vce_cs_parse - parse and validate the command stream
716 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
)
718 struct amdgpu_ib
*ib
= &p
->job
->ibs
[ib_idx
];
719 unsigned fb_idx
= 0, bs_idx
= 0;
720 int session_idx
= -1;
721 uint32_t destroyed
= 0;
722 uint32_t created
= 0;
723 uint32_t allocated
= 0;
724 uint32_t tmp
, handle
= 0;
725 uint32_t *size
= &tmp
;
730 ib
->gpu_addr
= amdgpu_sa_bo_gpu_addr(ib
->sa_bo
);
732 for (idx
= 0; idx
< ib
->length_dw
;) {
733 uint32_t len
= amdgpu_get_ib_value(p
, ib_idx
, idx
);
734 uint32_t cmd
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 1);
736 if ((len
< 8) || (len
& 3)) {
737 DRM_ERROR("invalid VCE command length (%d)!\n", len
);
743 case 0x00000002: /* task info */
744 fb_idx
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 6);
745 bs_idx
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 7);
748 case 0x03000001: /* encode */
749 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 10,
754 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 12,
760 case 0x05000001: /* context buffer */
761 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 3,
767 case 0x05000004: /* video bitstream buffer */
768 tmp
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 4);
769 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 3, idx
+ 2,
775 case 0x05000005: /* feedback buffer */
776 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 3, idx
+ 2,
782 case 0x0500000d: /* MV buffer */
783 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 3,
788 r
= amdgpu_vce_validate_bo(p
, ib_idx
, idx
+ 8,
798 for (idx
= 0; idx
< ib
->length_dw
;) {
799 uint32_t len
= amdgpu_get_ib_value(p
, ib_idx
, idx
);
800 uint32_t cmd
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 1);
803 case 0x00000001: /* session */
804 handle
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 2);
805 session_idx
= amdgpu_vce_validate_handle(p
, handle
,
807 if (session_idx
< 0) {
811 size
= &p
->adev
->vce
.img_size
[session_idx
];
814 case 0x00000002: /* task info */
815 fb_idx
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 6);
816 bs_idx
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 7);
819 case 0x01000001: /* create */
820 created
|= 1 << session_idx
;
821 if (destroyed
& (1 << session_idx
)) {
822 destroyed
&= ~(1 << session_idx
);
823 allocated
|= 1 << session_idx
;
825 } else if (!(allocated
& (1 << session_idx
))) {
826 DRM_ERROR("Handle already in use!\n");
831 *size
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 8) *
832 amdgpu_get_ib_value(p
, ib_idx
, idx
+ 10) *
836 case 0x04000001: /* config extension */
837 case 0x04000002: /* pic control */
838 case 0x04000005: /* rate control */
839 case 0x04000007: /* motion estimation */
840 case 0x04000008: /* rdo */
841 case 0x04000009: /* vui */
842 case 0x05000002: /* auxiliary buffer */
843 case 0x05000009: /* clock table */
846 case 0x0500000c: /* hw config */
847 switch (p
->adev
->asic_type
) {
848 #ifdef CONFIG_DRM_AMDGPU_CIK
860 case 0x03000001: /* encode */
861 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 10, idx
+ 9,
866 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 12, idx
+ 11,
872 case 0x02000001: /* destroy */
873 destroyed
|= 1 << session_idx
;
876 case 0x05000001: /* context buffer */
877 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 3, idx
+ 2,
883 case 0x05000004: /* video bitstream buffer */
884 tmp
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 4);
885 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 3, idx
+ 2,
891 case 0x05000005: /* feedback buffer */
892 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 3, idx
+ 2,
898 case 0x0500000d: /* MV buffer */
899 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 3,
904 r
= amdgpu_vce_cs_reloc(p
, ib_idx
, idx
+ 8,
905 idx
+ 7, *size
/ 12, 0);
911 DRM_ERROR("invalid VCE command (0x%x)!\n", cmd
);
916 if (session_idx
== -1) {
917 DRM_ERROR("no session command at start of IB\n");
925 if (allocated
& ~created
) {
926 DRM_ERROR("New session without create command!\n");
932 /* No error, free all destroyed handle slots */
935 /* Error during parsing, free all allocated handle slots */
939 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
)
941 atomic_set(&p
->adev
->vce
.handles
[i
], 0);
947 * amdgpu_vce_cs_parse_vm - parse the command stream in VM mode
952 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser
*p
, uint32_t ib_idx
)
954 struct amdgpu_ib
*ib
= &p
->job
->ibs
[ib_idx
];
955 int session_idx
= -1;
956 uint32_t destroyed
= 0;
957 uint32_t created
= 0;
958 uint32_t allocated
= 0;
959 uint32_t tmp
, handle
= 0;
960 int i
, r
= 0, idx
= 0;
962 while (idx
< ib
->length_dw
) {
963 uint32_t len
= amdgpu_get_ib_value(p
, ib_idx
, idx
);
964 uint32_t cmd
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 1);
966 if ((len
< 8) || (len
& 3)) {
967 DRM_ERROR("invalid VCE command length (%d)!\n", len
);
973 case 0x00000001: /* session */
974 handle
= amdgpu_get_ib_value(p
, ib_idx
, idx
+ 2);
975 session_idx
= amdgpu_vce_validate_handle(p
, handle
,
977 if (session_idx
< 0) {
983 case 0x01000001: /* create */
984 created
|= 1 << session_idx
;
985 if (destroyed
& (1 << session_idx
)) {
986 destroyed
&= ~(1 << session_idx
);
987 allocated
|= 1 << session_idx
;
989 } else if (!(allocated
& (1 << session_idx
))) {
990 DRM_ERROR("Handle already in use!\n");
997 case 0x02000001: /* destroy */
998 destroyed
|= 1 << session_idx
;
1005 if (session_idx
== -1) {
1006 DRM_ERROR("no session command at start of IB\n");
1014 if (allocated
& ~created
) {
1015 DRM_ERROR("New session without create command!\n");
1021 /* No error, free all destroyed handle slots */
1023 amdgpu_ib_free(p
->adev
, ib
, NULL
);
1025 /* Error during parsing, free all allocated handle slots */
1029 for (i
= 0; i
< AMDGPU_MAX_VCE_HANDLES
; ++i
)
1031 atomic_set(&p
->adev
->vce
.handles
[i
], 0);
1037 * amdgpu_vce_ring_emit_ib - execute indirect buffer
1039 * @ring: engine to use
1040 * @ib: the IB to execute
1043 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring
*ring
,
1044 struct amdgpu_job
*job
,
1045 struct amdgpu_ib
*ib
,
1048 amdgpu_ring_write(ring
, VCE_CMD_IB
);
1049 amdgpu_ring_write(ring
, lower_32_bits(ib
->gpu_addr
));
1050 amdgpu_ring_write(ring
, upper_32_bits(ib
->gpu_addr
));
1051 amdgpu_ring_write(ring
, ib
->length_dw
);
1055 * amdgpu_vce_ring_emit_fence - add a fence command to the ring
1057 * @ring: engine to use
1061 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring
*ring
, u64 addr
, u64 seq
,
1064 WARN_ON(flags
& AMDGPU_FENCE_FLAG_64BIT
);
1066 amdgpu_ring_write(ring
, VCE_CMD_FENCE
);
1067 amdgpu_ring_write(ring
, addr
);
1068 amdgpu_ring_write(ring
, upper_32_bits(addr
));
1069 amdgpu_ring_write(ring
, seq
);
1070 amdgpu_ring_write(ring
, VCE_CMD_TRAP
);
1071 amdgpu_ring_write(ring
, VCE_CMD_END
);
1075 * amdgpu_vce_ring_test_ring - test if VCE ring is working
1077 * @ring: the engine to test on
1080 int amdgpu_vce_ring_test_ring(struct amdgpu_ring
*ring
)
1082 struct amdgpu_device
*adev
= ring
->adev
;
1085 int r
, timeout
= adev
->usec_timeout
;
1087 /* skip ring test for sriov*/
1088 if (amdgpu_sriov_vf(adev
))
1091 r
= amdgpu_ring_alloc(ring
, 16);
1095 rptr
= amdgpu_ring_get_rptr(ring
);
1097 amdgpu_ring_write(ring
, VCE_CMD_END
);
1098 amdgpu_ring_commit(ring
);
1100 for (i
= 0; i
< timeout
; i
++) {
1101 if (amdgpu_ring_get_rptr(ring
) != rptr
)
1113 * amdgpu_vce_ring_test_ib - test if VCE IBs are working
1115 * @ring: the engine to test on
1118 int amdgpu_vce_ring_test_ib(struct amdgpu_ring
*ring
, long timeout
)
1120 struct dma_fence
*fence
= NULL
;
1121 struct amdgpu_bo
*bo
= NULL
;
1124 /* skip vce ring1/2 ib test for now, since it's not reliable */
1125 if (ring
!= &ring
->adev
->vce
.ring
[0])
1128 r
= amdgpu_bo_create_reserved(ring
->adev
, 512, PAGE_SIZE
,
1129 AMDGPU_GEM_DOMAIN_VRAM
,
1134 r
= amdgpu_vce_get_create_msg(ring
, 1, bo
, NULL
);
1138 r
= amdgpu_vce_get_destroy_msg(ring
, 1, true, &fence
);
1142 r
= dma_fence_wait_timeout(fence
, false, timeout
);
1149 dma_fence_put(fence
);
1150 amdgpu_bo_unreserve(bo
);
1151 amdgpu_bo_unref(&bo
);