1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/compat.h>
3 #include <linux/dma-mapping.h>
4 #include <linux/iommu.h>
5 #include <linux/module.h>
6 #include <linux/poll.h>
7 #include <linux/slab.h>
8 #include <linux/uacce.h>
10 static struct class *uacce_class
;
11 static dev_t uacce_devt
;
12 static DEFINE_MUTEX(uacce_mutex
);
13 static DEFINE_XARRAY_ALLOC(uacce_xa
);
15 static int uacce_start_queue(struct uacce_queue
*q
)
19 mutex_lock(&uacce_mutex
);
21 if (q
->state
!= UACCE_Q_INIT
) {
26 if (q
->uacce
->ops
->start_queue
) {
27 ret
= q
->uacce
->ops
->start_queue(q
);
32 q
->state
= UACCE_Q_STARTED
;
35 mutex_unlock(&uacce_mutex
);
40 static int uacce_put_queue(struct uacce_queue
*q
)
42 struct uacce_device
*uacce
= q
->uacce
;
44 mutex_lock(&uacce_mutex
);
46 if (q
->state
== UACCE_Q_ZOMBIE
)
49 if ((q
->state
== UACCE_Q_STARTED
) && uacce
->ops
->stop_queue
)
50 uacce
->ops
->stop_queue(q
);
52 if ((q
->state
== UACCE_Q_INIT
|| q
->state
== UACCE_Q_STARTED
) &&
53 uacce
->ops
->put_queue
)
54 uacce
->ops
->put_queue(q
);
56 q
->state
= UACCE_Q_ZOMBIE
;
58 mutex_unlock(&uacce_mutex
);
63 static long uacce_fops_unl_ioctl(struct file
*filep
,
64 unsigned int cmd
, unsigned long arg
)
66 struct uacce_queue
*q
= filep
->private_data
;
67 struct uacce_device
*uacce
= q
->uacce
;
70 case UACCE_CMD_START_Q
:
71 return uacce_start_queue(q
);
74 return uacce_put_queue(q
);
77 if (!uacce
->ops
->ioctl
)
80 return uacce
->ops
->ioctl(q
, cmd
, arg
);
85 static long uacce_fops_compat_ioctl(struct file
*filep
,
86 unsigned int cmd
, unsigned long arg
)
88 arg
= (unsigned long)compat_ptr(arg
);
90 return uacce_fops_unl_ioctl(filep
, cmd
, arg
);
94 static int uacce_bind_queue(struct uacce_device
*uacce
, struct uacce_queue
*q
)
97 struct iommu_sva
*handle
;
99 if (!(uacce
->flags
& UACCE_DEV_SVA
))
102 handle
= iommu_sva_bind_device(uacce
->parent
, current
->mm
, NULL
);
104 return PTR_ERR(handle
);
106 pasid
= iommu_sva_get_pasid(handle
);
107 if (pasid
== IOMMU_PASID_INVALID
) {
108 iommu_sva_unbind_device(handle
);
117 static void uacce_unbind_queue(struct uacce_queue
*q
)
121 iommu_sva_unbind_device(q
->handle
);
125 static int uacce_fops_open(struct inode
*inode
, struct file
*filep
)
127 struct uacce_device
*uacce
;
128 struct uacce_queue
*q
;
131 uacce
= xa_load(&uacce_xa
, iminor(inode
));
135 q
= kzalloc(sizeof(struct uacce_queue
), GFP_KERNEL
);
139 ret
= uacce_bind_queue(uacce
, q
);
145 if (uacce
->ops
->get_queue
) {
146 ret
= uacce
->ops
->get_queue(uacce
, q
->pasid
, q
);
151 init_waitqueue_head(&q
->wait
);
152 filep
->private_data
= q
;
153 uacce
->inode
= inode
;
154 q
->state
= UACCE_Q_INIT
;
156 mutex_lock(&uacce
->queues_lock
);
157 list_add(&q
->list
, &uacce
->queues
);
158 mutex_unlock(&uacce
->queues_lock
);
163 uacce_unbind_queue(q
);
169 static int uacce_fops_release(struct inode
*inode
, struct file
*filep
)
171 struct uacce_queue
*q
= filep
->private_data
;
173 mutex_lock(&q
->uacce
->queues_lock
);
175 mutex_unlock(&q
->uacce
->queues_lock
);
177 uacce_unbind_queue(q
);
183 static void uacce_vma_close(struct vm_area_struct
*vma
)
185 struct uacce_queue
*q
= vma
->vm_private_data
;
186 struct uacce_qfile_region
*qfr
= NULL
;
188 if (vma
->vm_pgoff
< UACCE_MAX_REGION
)
189 qfr
= q
->qfrs
[vma
->vm_pgoff
];
194 static const struct vm_operations_struct uacce_vm_ops
= {
195 .close
= uacce_vma_close
,
198 static int uacce_fops_mmap(struct file
*filep
, struct vm_area_struct
*vma
)
200 struct uacce_queue
*q
= filep
->private_data
;
201 struct uacce_device
*uacce
= q
->uacce
;
202 struct uacce_qfile_region
*qfr
;
203 enum uacce_qfrt type
= UACCE_MAX_REGION
;
206 if (vma
->vm_pgoff
< UACCE_MAX_REGION
)
207 type
= vma
->vm_pgoff
;
211 qfr
= kzalloc(sizeof(*qfr
), GFP_KERNEL
);
215 vma
->vm_flags
|= VM_DONTCOPY
| VM_DONTEXPAND
| VM_WIPEONFORK
;
216 vma
->vm_ops
= &uacce_vm_ops
;
217 vma
->vm_private_data
= q
;
220 mutex_lock(&uacce_mutex
);
222 if (q
->state
!= UACCE_Q_INIT
&& q
->state
!= UACCE_Q_STARTED
) {
233 case UACCE_QFRT_MMIO
:
235 if (!uacce
->ops
->mmap
) {
240 ret
= uacce
->ops
->mmap(q
, vma
, qfr
);
251 mutex_unlock(&uacce_mutex
);
256 mutex_unlock(&uacce_mutex
);
261 static __poll_t
uacce_fops_poll(struct file
*file
, poll_table
*wait
)
263 struct uacce_queue
*q
= file
->private_data
;
264 struct uacce_device
*uacce
= q
->uacce
;
266 poll_wait(file
, &q
->wait
, wait
);
267 if (uacce
->ops
->is_q_updated
&& uacce
->ops
->is_q_updated(q
))
268 return EPOLLIN
| EPOLLRDNORM
;
273 static const struct file_operations uacce_fops
= {
274 .owner
= THIS_MODULE
,
275 .open
= uacce_fops_open
,
276 .release
= uacce_fops_release
,
277 .unlocked_ioctl
= uacce_fops_unl_ioctl
,
279 .compat_ioctl
= uacce_fops_compat_ioctl
,
281 .mmap
= uacce_fops_mmap
,
282 .poll
= uacce_fops_poll
,
285 #define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
287 static ssize_t
api_show(struct device
*dev
,
288 struct device_attribute
*attr
, char *buf
)
290 struct uacce_device
*uacce
= to_uacce_device(dev
);
292 return sprintf(buf
, "%s\n", uacce
->api_ver
);
295 static ssize_t
flags_show(struct device
*dev
,
296 struct device_attribute
*attr
, char *buf
)
298 struct uacce_device
*uacce
= to_uacce_device(dev
);
300 return sprintf(buf
, "%u\n", uacce
->flags
);
303 static ssize_t
available_instances_show(struct device
*dev
,
304 struct device_attribute
*attr
,
307 struct uacce_device
*uacce
= to_uacce_device(dev
);
309 if (!uacce
->ops
->get_available_instances
)
312 return sprintf(buf
, "%d\n",
313 uacce
->ops
->get_available_instances(uacce
));
316 static ssize_t
algorithms_show(struct device
*dev
,
317 struct device_attribute
*attr
, char *buf
)
319 struct uacce_device
*uacce
= to_uacce_device(dev
);
321 return sprintf(buf
, "%s\n", uacce
->algs
);
324 static ssize_t
region_mmio_size_show(struct device
*dev
,
325 struct device_attribute
*attr
, char *buf
)
327 struct uacce_device
*uacce
= to_uacce_device(dev
);
329 return sprintf(buf
, "%lu\n",
330 uacce
->qf_pg_num
[UACCE_QFRT_MMIO
] << PAGE_SHIFT
);
333 static ssize_t
region_dus_size_show(struct device
*dev
,
334 struct device_attribute
*attr
, char *buf
)
336 struct uacce_device
*uacce
= to_uacce_device(dev
);
338 return sprintf(buf
, "%lu\n",
339 uacce
->qf_pg_num
[UACCE_QFRT_DUS
] << PAGE_SHIFT
);
342 static DEVICE_ATTR_RO(api
);
343 static DEVICE_ATTR_RO(flags
);
344 static DEVICE_ATTR_RO(available_instances
);
345 static DEVICE_ATTR_RO(algorithms
);
346 static DEVICE_ATTR_RO(region_mmio_size
);
347 static DEVICE_ATTR_RO(region_dus_size
);
349 static struct attribute
*uacce_dev_attrs
[] = {
351 &dev_attr_flags
.attr
,
352 &dev_attr_available_instances
.attr
,
353 &dev_attr_algorithms
.attr
,
354 &dev_attr_region_mmio_size
.attr
,
355 &dev_attr_region_dus_size
.attr
,
359 static umode_t
uacce_dev_is_visible(struct kobject
*kobj
,
360 struct attribute
*attr
, int n
)
362 struct device
*dev
= kobj_to_dev(kobj
);
363 struct uacce_device
*uacce
= to_uacce_device(dev
);
365 if (((attr
== &dev_attr_region_mmio_size
.attr
) &&
366 (!uacce
->qf_pg_num
[UACCE_QFRT_MMIO
])) ||
367 ((attr
== &dev_attr_region_dus_size
.attr
) &&
368 (!uacce
->qf_pg_num
[UACCE_QFRT_DUS
])))
374 static struct attribute_group uacce_dev_group
= {
375 .is_visible
= uacce_dev_is_visible
,
376 .attrs
= uacce_dev_attrs
,
379 __ATTRIBUTE_GROUPS(uacce_dev
);
381 static void uacce_release(struct device
*dev
)
383 struct uacce_device
*uacce
= to_uacce_device(dev
);
389 * uacce_alloc() - alloc an accelerator
390 * @parent: pointer of uacce parent device
391 * @interface: pointer of uacce_interface for register
393 * Returns uacce pointer if success and ERR_PTR if not
394 * Need check returned negotiated uacce->flags
396 struct uacce_device
*uacce_alloc(struct device
*parent
,
397 struct uacce_interface
*interface
)
399 unsigned int flags
= interface
->flags
;
400 struct uacce_device
*uacce
;
403 uacce
= kzalloc(sizeof(struct uacce_device
), GFP_KERNEL
);
405 return ERR_PTR(-ENOMEM
);
407 if (flags
& UACCE_DEV_SVA
) {
408 ret
= iommu_dev_enable_feature(parent
, IOMMU_DEV_FEAT_SVA
);
410 flags
&= ~UACCE_DEV_SVA
;
413 uacce
->parent
= parent
;
414 uacce
->flags
= flags
;
415 uacce
->ops
= interface
->ops
;
417 ret
= xa_alloc(&uacce_xa
, &uacce
->dev_id
, uacce
, xa_limit_32b
,
422 INIT_LIST_HEAD(&uacce
->queues
);
423 mutex_init(&uacce
->queues_lock
);
424 device_initialize(&uacce
->dev
);
425 uacce
->dev
.devt
= MKDEV(MAJOR(uacce_devt
), uacce
->dev_id
);
426 uacce
->dev
.class = uacce_class
;
427 uacce
->dev
.groups
= uacce_dev_groups
;
428 uacce
->dev
.parent
= uacce
->parent
;
429 uacce
->dev
.release
= uacce_release
;
430 dev_set_name(&uacce
->dev
, "%s-%d", interface
->name
, uacce
->dev_id
);
435 if (flags
& UACCE_DEV_SVA
)
436 iommu_dev_disable_feature(uacce
->parent
, IOMMU_DEV_FEAT_SVA
);
440 EXPORT_SYMBOL_GPL(uacce_alloc
);
443 * uacce_register() - add the accelerator to cdev and export to user space
444 * @uacce: The initialized uacce device
446 * Return 0 if register succeeded, or an error.
448 int uacce_register(struct uacce_device
*uacce
)
453 uacce
->cdev
= cdev_alloc();
457 uacce
->cdev
->ops
= &uacce_fops
;
458 uacce
->cdev
->owner
= THIS_MODULE
;
460 return cdev_device_add(uacce
->cdev
, &uacce
->dev
);
462 EXPORT_SYMBOL_GPL(uacce_register
);
465 * uacce_remove() - remove the accelerator
466 * @uacce: the accelerator to remove
468 void uacce_remove(struct uacce_device
*uacce
)
470 struct uacce_queue
*q
, *next_q
;
475 * unmap remaining mapping from user space, preventing user still
476 * access the mmaped area while parent device is already removed
479 unmap_mapping_range(uacce
->inode
->i_mapping
, 0, 0, 1);
481 /* ensure no open queue remains */
482 mutex_lock(&uacce
->queues_lock
);
483 list_for_each_entry_safe(q
, next_q
, &uacce
->queues
, list
) {
485 uacce_unbind_queue(q
);
487 mutex_unlock(&uacce
->queues_lock
);
489 /* disable sva now since no opened queues */
490 if (uacce
->flags
& UACCE_DEV_SVA
)
491 iommu_dev_disable_feature(uacce
->parent
, IOMMU_DEV_FEAT_SVA
);
494 cdev_device_del(uacce
->cdev
, &uacce
->dev
);
495 xa_erase(&uacce_xa
, uacce
->dev_id
);
496 put_device(&uacce
->dev
);
498 EXPORT_SYMBOL_GPL(uacce_remove
);
500 static int __init
uacce_init(void)
504 uacce_class
= class_create(THIS_MODULE
, UACCE_NAME
);
505 if (IS_ERR(uacce_class
))
506 return PTR_ERR(uacce_class
);
508 ret
= alloc_chrdev_region(&uacce_devt
, 0, MINORMASK
, UACCE_NAME
);
510 class_destroy(uacce_class
);
515 static __exit
void uacce_exit(void)
517 unregister_chrdev_region(uacce_devt
, MINORMASK
);
518 class_destroy(uacce_class
);
521 subsys_initcall(uacce_init
);
522 module_exit(uacce_exit
);
524 MODULE_LICENSE("GPL");
525 MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
526 MODULE_DESCRIPTION("Accelerator interface for Userland applications");