1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2015, Linaro Limited
5 #include <linux/arm-smccc.h>
6 #include <linux/device.h>
8 #include <linux/errno.h>
10 #include <linux/slab.h>
11 #include <linux/tee_drv.h>
12 #include <linux/types.h>
13 #include <linux/uaccess.h>
14 #include "optee_private.h"
15 #include "optee_smc.h"
17 struct optee_call_waiter
{
18 struct list_head list_node
;
22 static void optee_cq_wait_init(struct optee_call_queue
*cq
,
23 struct optee_call_waiter
*w
)
26 * We're preparing to make a call to secure world. In case we can't
27 * allocate a thread in secure world we'll end up waiting in
28 * optee_cq_wait_for_completion().
30 * Normally if there's no contention in secure world the call will
31 * complete and we can cleanup directly with optee_cq_wait_final().
33 mutex_lock(&cq
->mutex
);
36 * We add ourselves to the queue, but we don't wait. This
37 * guarantees that we don't lose a completion if secure world
38 * returns busy and another thread just exited and try to complete
41 init_completion(&w
->c
);
42 list_add_tail(&w
->list_node
, &cq
->waiters
);
44 mutex_unlock(&cq
->mutex
);
47 static void optee_cq_wait_for_completion(struct optee_call_queue
*cq
,
48 struct optee_call_waiter
*w
)
50 wait_for_completion(&w
->c
);
52 mutex_lock(&cq
->mutex
);
54 /* Move to end of list to get out of the way for other waiters */
55 list_del(&w
->list_node
);
56 reinit_completion(&w
->c
);
57 list_add_tail(&w
->list_node
, &cq
->waiters
);
59 mutex_unlock(&cq
->mutex
);
62 static void optee_cq_complete_one(struct optee_call_queue
*cq
)
64 struct optee_call_waiter
*w
;
66 list_for_each_entry(w
, &cq
->waiters
, list_node
) {
67 if (!completion_done(&w
->c
)) {
74 static void optee_cq_wait_final(struct optee_call_queue
*cq
,
75 struct optee_call_waiter
*w
)
78 * We're done with the call to secure world. The thread in secure
79 * world that was used for this call is now available for some
82 mutex_lock(&cq
->mutex
);
84 /* Get out of the list */
85 list_del(&w
->list_node
);
87 /* Wake up one eventual waiting task */
88 optee_cq_complete_one(cq
);
91 * If we're completed we've got a completion from another task that
92 * was just done with its call to secure world. Since yet another
93 * thread now is available in secure world wake up another eventual
96 if (completion_done(&w
->c
))
97 optee_cq_complete_one(cq
);
99 mutex_unlock(&cq
->mutex
);
102 /* Requires the filpstate mutex to be held */
103 static struct optee_session
*find_session(struct optee_context_data
*ctxdata
,
106 struct optee_session
*sess
;
108 list_for_each_entry(sess
, &ctxdata
->sess_list
, list_node
)
109 if (sess
->session_id
== session_id
)
116 * optee_do_call_with_arg() - Do an SMC to OP-TEE in secure world
117 * @ctx: calling context
118 * @parg: physical address of message to pass to secure world
120 * Does and SMC to OP-TEE in secure world and handles eventual resulting
121 * Remote Procedure Calls (RPC) from OP-TEE.
123 * Returns return code from secure world, 0 is OK
125 u32
optee_do_call_with_arg(struct tee_context
*ctx
, phys_addr_t parg
)
127 struct optee
*optee
= tee_get_drvdata(ctx
->teedev
);
128 struct optee_call_waiter w
;
129 struct optee_rpc_param param
= { };
130 struct optee_call_ctx call_ctx
= { };
133 param
.a0
= OPTEE_SMC_CALL_WITH_ARG
;
134 reg_pair_from_64(¶m
.a1
, ¶m
.a2
, parg
);
135 /* Initialize waiter */
136 optee_cq_wait_init(&optee
->call_queue
, &w
);
138 struct arm_smccc_res res
;
140 optee
->invoke_fn(param
.a0
, param
.a1
, param
.a2
, param
.a3
,
141 param
.a4
, param
.a5
, param
.a6
, param
.a7
,
144 if (res
.a0
== OPTEE_SMC_RETURN_ETHREAD_LIMIT
) {
146 * Out of threads in secure world, wait for a thread
149 optee_cq_wait_for_completion(&optee
->call_queue
, &w
);
150 } else if (OPTEE_SMC_RETURN_IS_RPC(res
.a0
)) {
156 optee_handle_rpc(ctx
, ¶m
, &call_ctx
);
163 optee_rpc_finalize_call(&call_ctx
);
165 * We're done with our thread in secure world, if there's any
166 * thread waiters wake up one.
168 optee_cq_wait_final(&optee
->call_queue
, &w
);
173 static struct tee_shm
*get_msg_arg(struct tee_context
*ctx
, size_t num_params
,
174 struct optee_msg_arg
**msg_arg
,
175 phys_addr_t
*msg_parg
)
179 struct optee_msg_arg
*ma
;
181 shm
= tee_shm_alloc(ctx
, OPTEE_MSG_GET_ARG_SIZE(num_params
),
186 ma
= tee_shm_get_va(shm
, 0);
192 rc
= tee_shm_get_pa(shm
, 0, msg_parg
);
196 memset(ma
, 0, OPTEE_MSG_GET_ARG_SIZE(num_params
));
197 ma
->num_params
= num_params
;
208 int optee_open_session(struct tee_context
*ctx
,
209 struct tee_ioctl_open_session_arg
*arg
,
210 struct tee_param
*param
)
212 struct optee_context_data
*ctxdata
= ctx
->data
;
215 struct optee_msg_arg
*msg_arg
;
216 phys_addr_t msg_parg
;
217 struct optee_session
*sess
= NULL
;
219 /* +2 for the meta parameters added below */
220 shm
= get_msg_arg(ctx
, arg
->num_params
+ 2, &msg_arg
, &msg_parg
);
224 msg_arg
->cmd
= OPTEE_MSG_CMD_OPEN_SESSION
;
225 msg_arg
->cancel_id
= arg
->cancel_id
;
228 * Initialize and add the meta parameters needed when opening a
231 msg_arg
->params
[0].attr
= OPTEE_MSG_ATTR_TYPE_VALUE_INPUT
|
233 msg_arg
->params
[1].attr
= OPTEE_MSG_ATTR_TYPE_VALUE_INPUT
|
235 memcpy(&msg_arg
->params
[0].u
.value
, arg
->uuid
, sizeof(arg
->uuid
));
236 memcpy(&msg_arg
->params
[1].u
.value
, arg
->uuid
, sizeof(arg
->clnt_uuid
));
237 msg_arg
->params
[1].u
.value
.c
= arg
->clnt_login
;
239 rc
= optee_to_msg_param(msg_arg
->params
+ 2, arg
->num_params
, param
);
243 sess
= kzalloc(sizeof(*sess
), GFP_KERNEL
);
249 if (optee_do_call_with_arg(ctx
, msg_parg
)) {
250 msg_arg
->ret
= TEEC_ERROR_COMMUNICATION
;
251 msg_arg
->ret_origin
= TEEC_ORIGIN_COMMS
;
254 if (msg_arg
->ret
== TEEC_SUCCESS
) {
255 /* A new session has been created, add it to the list. */
256 sess
->session_id
= msg_arg
->session
;
257 mutex_lock(&ctxdata
->mutex
);
258 list_add(&sess
->list_node
, &ctxdata
->sess_list
);
259 mutex_unlock(&ctxdata
->mutex
);
264 if (optee_from_msg_param(param
, arg
->num_params
, msg_arg
->params
+ 2)) {
265 arg
->ret
= TEEC_ERROR_COMMUNICATION
;
266 arg
->ret_origin
= TEEC_ORIGIN_COMMS
;
267 /* Close session again to avoid leakage */
268 optee_close_session(ctx
, msg_arg
->session
);
270 arg
->session
= msg_arg
->session
;
271 arg
->ret
= msg_arg
->ret
;
272 arg
->ret_origin
= msg_arg
->ret_origin
;
280 int optee_close_session(struct tee_context
*ctx
, u32 session
)
282 struct optee_context_data
*ctxdata
= ctx
->data
;
284 struct optee_msg_arg
*msg_arg
;
285 phys_addr_t msg_parg
;
286 struct optee_session
*sess
;
288 /* Check that the session is valid and remove it from the list */
289 mutex_lock(&ctxdata
->mutex
);
290 sess
= find_session(ctxdata
, session
);
292 list_del(&sess
->list_node
);
293 mutex_unlock(&ctxdata
->mutex
);
298 shm
= get_msg_arg(ctx
, 0, &msg_arg
, &msg_parg
);
302 msg_arg
->cmd
= OPTEE_MSG_CMD_CLOSE_SESSION
;
303 msg_arg
->session
= session
;
304 optee_do_call_with_arg(ctx
, msg_parg
);
310 int optee_invoke_func(struct tee_context
*ctx
, struct tee_ioctl_invoke_arg
*arg
,
311 struct tee_param
*param
)
313 struct optee_context_data
*ctxdata
= ctx
->data
;
315 struct optee_msg_arg
*msg_arg
;
316 phys_addr_t msg_parg
;
317 struct optee_session
*sess
;
320 /* Check that the session is valid */
321 mutex_lock(&ctxdata
->mutex
);
322 sess
= find_session(ctxdata
, arg
->session
);
323 mutex_unlock(&ctxdata
->mutex
);
327 shm
= get_msg_arg(ctx
, arg
->num_params
, &msg_arg
, &msg_parg
);
330 msg_arg
->cmd
= OPTEE_MSG_CMD_INVOKE_COMMAND
;
331 msg_arg
->func
= arg
->func
;
332 msg_arg
->session
= arg
->session
;
333 msg_arg
->cancel_id
= arg
->cancel_id
;
335 rc
= optee_to_msg_param(msg_arg
->params
, arg
->num_params
, param
);
339 if (optee_do_call_with_arg(ctx
, msg_parg
)) {
340 msg_arg
->ret
= TEEC_ERROR_COMMUNICATION
;
341 msg_arg
->ret_origin
= TEEC_ORIGIN_COMMS
;
344 if (optee_from_msg_param(param
, arg
->num_params
, msg_arg
->params
)) {
345 msg_arg
->ret
= TEEC_ERROR_COMMUNICATION
;
346 msg_arg
->ret_origin
= TEEC_ORIGIN_COMMS
;
349 arg
->ret
= msg_arg
->ret
;
350 arg
->ret_origin
= msg_arg
->ret_origin
;
356 int optee_cancel_req(struct tee_context
*ctx
, u32 cancel_id
, u32 session
)
358 struct optee_context_data
*ctxdata
= ctx
->data
;
360 struct optee_msg_arg
*msg_arg
;
361 phys_addr_t msg_parg
;
362 struct optee_session
*sess
;
364 /* Check that the session is valid */
365 mutex_lock(&ctxdata
->mutex
);
366 sess
= find_session(ctxdata
, session
);
367 mutex_unlock(&ctxdata
->mutex
);
371 shm
= get_msg_arg(ctx
, 0, &msg_arg
, &msg_parg
);
375 msg_arg
->cmd
= OPTEE_MSG_CMD_CANCEL
;
376 msg_arg
->session
= session
;
377 msg_arg
->cancel_id
= cancel_id
;
378 optee_do_call_with_arg(ctx
, msg_parg
);
385 * optee_enable_shm_cache() - Enables caching of some shared memory allocation
387 * @optee: main service struct
389 void optee_enable_shm_cache(struct optee
*optee
)
391 struct optee_call_waiter w
;
393 /* We need to retry until secure world isn't busy. */
394 optee_cq_wait_init(&optee
->call_queue
, &w
);
396 struct arm_smccc_res res
;
398 optee
->invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE
, 0, 0, 0, 0, 0, 0,
400 if (res
.a0
== OPTEE_SMC_RETURN_OK
)
402 optee_cq_wait_for_completion(&optee
->call_queue
, &w
);
404 optee_cq_wait_final(&optee
->call_queue
, &w
);
408 * optee_disable_shm_cache() - Disables caching of some shared memory allocation
410 * @optee: main service struct
412 void optee_disable_shm_cache(struct optee
*optee
)
414 struct optee_call_waiter w
;
416 /* We need to retry until secure world isn't busy. */
417 optee_cq_wait_init(&optee
->call_queue
, &w
);
420 struct arm_smccc_res smccc
;
421 struct optee_smc_disable_shm_cache_result result
;
424 optee
->invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE
, 0, 0, 0, 0, 0, 0,
426 if (res
.result
.status
== OPTEE_SMC_RETURN_ENOTAVAIL
)
427 break; /* All shm's freed */
428 if (res
.result
.status
== OPTEE_SMC_RETURN_OK
) {
431 shm
= reg_pair_to_ptr(res
.result
.shm_upper32
,
432 res
.result
.shm_lower32
);
435 optee_cq_wait_for_completion(&optee
->call_queue
, &w
);
438 optee_cq_wait_final(&optee
->call_queue
, &w
);
441 #define PAGELIST_ENTRIES_PER_PAGE \
442 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
445 * optee_fill_pages_list() - write list of user pages to given shared
448 * @dst: page-aligned buffer where list of pages will be stored
449 * @pages: array of pages that represents shared buffer
450 * @num_pages: number of entries in @pages
451 * @page_offset: offset of user buffer from page start
453 * @dst should be big enough to hold list of user page addresses and
454 * links to the next pages of buffer
456 void optee_fill_pages_list(u64
*dst
, struct page
**pages
, int num_pages
,
460 phys_addr_t optee_page
;
462 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
466 u64 pages_list
[PAGELIST_ENTRIES_PER_PAGE
];
471 * Currently OP-TEE uses 4k page size and it does not looks
472 * like this will change in the future. On other hand, there are
473 * no know ARM architectures with page size < 4k.
474 * Thus the next built assert looks redundant. But the following
475 * code heavily relies on this assumption, so it is better be
478 BUILD_BUG_ON(PAGE_SIZE
< OPTEE_MSG_NONCONTIG_PAGE_SIZE
);
480 pages_data
= (void *)dst
;
482 * If linux page is bigger than 4k, and user buffer offset is
483 * larger than 4k/8k/12k/etc this will skip first 4k pages,
484 * because they bear no value data for OP-TEE.
486 optee_page
= page_to_phys(*pages
) +
487 round_down(page_offset
, OPTEE_MSG_NONCONTIG_PAGE_SIZE
);
490 pages_data
->pages_list
[n
++] = optee_page
;
492 if (n
== PAGELIST_ENTRIES_PER_PAGE
) {
493 pages_data
->next_page_data
=
494 virt_to_phys(pages_data
+ 1);
499 optee_page
+= OPTEE_MSG_NONCONTIG_PAGE_SIZE
;
500 if (!(optee_page
& ~PAGE_MASK
)) {
504 optee_page
= page_to_phys(*pages
);
510 * The final entry in each pagelist page is a pointer to the next
513 static size_t get_pages_list_size(size_t num_entries
)
515 int pages
= DIV_ROUND_UP(num_entries
, PAGELIST_ENTRIES_PER_PAGE
);
517 return pages
* OPTEE_MSG_NONCONTIG_PAGE_SIZE
;
520 u64
*optee_allocate_pages_list(size_t num_entries
)
522 return alloc_pages_exact(get_pages_list_size(num_entries
), GFP_KERNEL
);
525 void optee_free_pages_list(void *list
, size_t num_entries
)
527 free_pages_exact(list
, get_pages_list_size(num_entries
));
530 static bool is_normal_memory(pgprot_t p
)
532 #if defined(CONFIG_ARM)
533 return (pgprot_val(p
) & L_PTE_MT_MASK
) == L_PTE_MT_WRITEALLOC
;
534 #elif defined(CONFIG_ARM64)
535 return (pgprot_val(p
) & PTE_ATTRINDX_MASK
) == PTE_ATTRINDX(MT_NORMAL
);
537 #error "Unuspported architecture"
541 static int __check_mem_type(struct vm_area_struct
*vma
, unsigned long end
)
543 while (vma
&& is_normal_memory(vma
->vm_page_prot
)) {
544 if (vma
->vm_end
>= end
)
552 static int check_mem_type(unsigned long start
, size_t num_pages
)
554 struct mm_struct
*mm
= current
->mm
;
558 * Allow kernel address to register with OP-TEE as kernel
559 * pages are configured as normal memory only.
561 if (virt_addr_valid(start
))
564 down_read(&mm
->mmap_sem
);
565 rc
= __check_mem_type(find_vma(mm
, start
),
566 start
+ num_pages
* PAGE_SIZE
);
567 up_read(&mm
->mmap_sem
);
572 int optee_shm_register(struct tee_context
*ctx
, struct tee_shm
*shm
,
573 struct page
**pages
, size_t num_pages
,
576 struct tee_shm
*shm_arg
= NULL
;
577 struct optee_msg_arg
*msg_arg
;
579 phys_addr_t msg_parg
;
585 rc
= check_mem_type(start
, num_pages
);
589 pages_list
= optee_allocate_pages_list(num_pages
);
593 shm_arg
= get_msg_arg(ctx
, 1, &msg_arg
, &msg_parg
);
594 if (IS_ERR(shm_arg
)) {
595 rc
= PTR_ERR(shm_arg
);
599 optee_fill_pages_list(pages_list
, pages
, num_pages
,
600 tee_shm_get_page_offset(shm
));
602 msg_arg
->cmd
= OPTEE_MSG_CMD_REGISTER_SHM
;
603 msg_arg
->params
->attr
= OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT
|
604 OPTEE_MSG_ATTR_NONCONTIG
;
605 msg_arg
->params
->u
.tmem
.shm_ref
= (unsigned long)shm
;
606 msg_arg
->params
->u
.tmem
.size
= tee_shm_get_size(shm
);
608 * In the least bits of msg_arg->params->u.tmem.buf_ptr we
609 * store buffer offset from 4k page, as described in OP-TEE ABI.
611 msg_arg
->params
->u
.tmem
.buf_ptr
= virt_to_phys(pages_list
) |
612 (tee_shm_get_page_offset(shm
) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE
- 1));
614 if (optee_do_call_with_arg(ctx
, msg_parg
) ||
615 msg_arg
->ret
!= TEEC_SUCCESS
)
618 tee_shm_free(shm_arg
);
620 optee_free_pages_list(pages_list
, num_pages
);
624 int optee_shm_unregister(struct tee_context
*ctx
, struct tee_shm
*shm
)
626 struct tee_shm
*shm_arg
;
627 struct optee_msg_arg
*msg_arg
;
628 phys_addr_t msg_parg
;
631 shm_arg
= get_msg_arg(ctx
, 1, &msg_arg
, &msg_parg
);
633 return PTR_ERR(shm_arg
);
635 msg_arg
->cmd
= OPTEE_MSG_CMD_UNREGISTER_SHM
;
637 msg_arg
->params
[0].attr
= OPTEE_MSG_ATTR_TYPE_RMEM_INPUT
;
638 msg_arg
->params
[0].u
.rmem
.shm_ref
= (unsigned long)shm
;
640 if (optee_do_call_with_arg(ctx
, msg_parg
) ||
641 msg_arg
->ret
!= TEEC_SUCCESS
)
643 tee_shm_free(shm_arg
);
647 int optee_shm_register_supp(struct tee_context
*ctx
, struct tee_shm
*shm
,
648 struct page
**pages
, size_t num_pages
,
652 * We don't want to register supplicant memory in OP-TEE.
653 * Instead information about it will be passed in RPC code.
655 return check_mem_type(start
, num_pages
);
658 int optee_shm_unregister_supp(struct tee_context
*ctx
, struct tee_shm
*shm
)