1 // SPDX-License-Identifier: GPL-2.0
3 * Virtio driver for the paravirtualized IOMMU
5 * Copyright (C) 2019 Arm Limited
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 #include <linux/amba/bus.h>
11 #include <linux/delay.h>
12 #include <linux/dma-iommu.h>
13 #include <linux/freezer.h>
14 #include <linux/interval_tree.h>
15 #include <linux/iommu.h>
16 #include <linux/module.h>
17 #include <linux/of_iommu.h>
18 #include <linux/of_platform.h>
19 #include <linux/pci.h>
20 #include <linux/platform_device.h>
21 #include <linux/virtio.h>
22 #include <linux/virtio_config.h>
23 #include <linux/virtio_ids.h>
24 #include <linux/wait.h>
26 #include <uapi/linux/virtio_iommu.h>
28 #define MSI_IOVA_BASE 0x8000000
29 #define MSI_IOVA_LENGTH 0x100000
31 #define VIOMMU_REQUEST_VQ 0
32 #define VIOMMU_EVENT_VQ 1
33 #define VIOMMU_NR_VQS 2
36 struct iommu_device iommu
;
38 struct virtio_device
*vdev
;
40 struct ida domain_ids
;
42 struct virtqueue
*vqs
[VIOMMU_NR_VQS
];
43 spinlock_t request_lock
;
44 struct list_head requests
;
47 /* Device configuration */
48 struct iommu_domain_geometry geometry
;
52 /* Supported MAP flags */
57 struct viommu_mapping
{
59 struct interval_tree_node iova
;
63 struct viommu_domain
{
64 struct iommu_domain domain
;
65 struct viommu_dev
*viommu
;
66 struct mutex mutex
; /* protects viommu pointer */
70 spinlock_t mappings_lock
;
71 struct rb_root_cached mappings
;
73 unsigned long nr_endpoints
;
76 struct viommu_endpoint
{
78 struct viommu_dev
*viommu
;
79 struct viommu_domain
*vdomain
;
80 struct list_head resv_regions
;
83 struct viommu_request
{
84 struct list_head list
;
86 unsigned int write_offset
;
91 #define VIOMMU_FAULT_RESV_MASK 0xffffff00
96 struct virtio_iommu_fault fault
;
100 #define to_viommu_domain(domain) \
101 container_of(domain, struct viommu_domain, domain)
103 static int viommu_get_req_errno(void *buf
, size_t len
)
105 struct virtio_iommu_req_tail
*tail
= buf
+ len
- sizeof(*tail
);
107 switch (tail
->status
) {
108 case VIRTIO_IOMMU_S_OK
:
110 case VIRTIO_IOMMU_S_UNSUPP
:
112 case VIRTIO_IOMMU_S_INVAL
:
114 case VIRTIO_IOMMU_S_RANGE
:
116 case VIRTIO_IOMMU_S_NOENT
:
118 case VIRTIO_IOMMU_S_FAULT
:
120 case VIRTIO_IOMMU_S_NOMEM
:
122 case VIRTIO_IOMMU_S_IOERR
:
123 case VIRTIO_IOMMU_S_DEVERR
:
129 static void viommu_set_req_status(void *buf
, size_t len
, int status
)
131 struct virtio_iommu_req_tail
*tail
= buf
+ len
- sizeof(*tail
);
133 tail
->status
= status
;
136 static off_t
viommu_get_write_desc_offset(struct viommu_dev
*viommu
,
137 struct virtio_iommu_req_head
*req
,
140 size_t tail_size
= sizeof(struct virtio_iommu_req_tail
);
142 if (req
->type
== VIRTIO_IOMMU_T_PROBE
)
143 return len
- viommu
->probe_size
- tail_size
;
145 return len
- tail_size
;
149 * __viommu_sync_req - Complete all in-flight requests
151 * Wait for all added requests to complete. When this function returns, all
152 * requests that were in-flight at the time of the call have completed.
154 static int __viommu_sync_req(struct viommu_dev
*viommu
)
159 struct viommu_request
*req
;
160 struct virtqueue
*vq
= viommu
->vqs
[VIOMMU_REQUEST_VQ
];
162 assert_spin_locked(&viommu
->request_lock
);
166 while (!list_empty(&viommu
->requests
)) {
168 req
= virtqueue_get_buf(vq
, &len
);
173 viommu_set_req_status(req
->buf
, req
->len
,
174 VIRTIO_IOMMU_S_IOERR
);
176 write_len
= req
->len
- req
->write_offset
;
177 if (req
->writeback
&& len
== write_len
)
178 memcpy(req
->writeback
, req
->buf
+ req
->write_offset
,
181 list_del(&req
->list
);
188 static int viommu_sync_req(struct viommu_dev
*viommu
)
193 spin_lock_irqsave(&viommu
->request_lock
, flags
);
194 ret
= __viommu_sync_req(viommu
);
196 dev_dbg(viommu
->dev
, "could not sync requests (%d)\n", ret
);
197 spin_unlock_irqrestore(&viommu
->request_lock
, flags
);
203 * __viommu_add_request - Add one request to the queue
204 * @buf: pointer to the request buffer
205 * @len: length of the request buffer
206 * @writeback: copy data back to the buffer when the request completes.
208 * Add a request to the queue. Only synchronize the queue if it's already full.
209 * Otherwise don't kick the queue nor wait for requests to complete.
211 * When @writeback is true, data written by the device, including the request
212 * status, is copied into @buf after the request completes. This is unsafe if
213 * the caller allocates @buf on stack and drops the lock between add_req() and
216 * Return 0 if the request was successfully added to the queue.
218 static int __viommu_add_req(struct viommu_dev
*viommu
, void *buf
, size_t len
,
223 struct viommu_request
*req
;
224 struct scatterlist top_sg
, bottom_sg
;
225 struct scatterlist
*sg
[2] = { &top_sg
, &bottom_sg
};
226 struct virtqueue
*vq
= viommu
->vqs
[VIOMMU_REQUEST_VQ
];
228 assert_spin_locked(&viommu
->request_lock
);
230 write_offset
= viommu_get_write_desc_offset(viommu
, buf
, len
);
231 if (write_offset
<= 0)
234 req
= kzalloc(sizeof(*req
) + len
, GFP_ATOMIC
);
240 req
->writeback
= buf
+ write_offset
;
241 req
->write_offset
= write_offset
;
243 memcpy(&req
->buf
, buf
, write_offset
);
245 sg_init_one(&top_sg
, req
->buf
, write_offset
);
246 sg_init_one(&bottom_sg
, req
->buf
+ write_offset
, len
- write_offset
);
248 ret
= virtqueue_add_sgs(vq
, sg
, 1, 1, req
, GFP_ATOMIC
);
249 if (ret
== -ENOSPC
) {
250 /* If the queue is full, sync and retry */
251 if (!__viommu_sync_req(viommu
))
252 ret
= virtqueue_add_sgs(vq
, sg
, 1, 1, req
, GFP_ATOMIC
);
257 list_add_tail(&req
->list
, &viommu
->requests
);
265 static int viommu_add_req(struct viommu_dev
*viommu
, void *buf
, size_t len
)
270 spin_lock_irqsave(&viommu
->request_lock
, flags
);
271 ret
= __viommu_add_req(viommu
, buf
, len
, false);
273 dev_dbg(viommu
->dev
, "could not add request: %d\n", ret
);
274 spin_unlock_irqrestore(&viommu
->request_lock
, flags
);
280 * Send a request and wait for it to complete. Return the request status (as an
283 static int viommu_send_req_sync(struct viommu_dev
*viommu
, void *buf
,
289 spin_lock_irqsave(&viommu
->request_lock
, flags
);
291 ret
= __viommu_add_req(viommu
, buf
, len
, true);
293 dev_dbg(viommu
->dev
, "could not add request (%d)\n", ret
);
297 ret
= __viommu_sync_req(viommu
);
299 dev_dbg(viommu
->dev
, "could not sync requests (%d)\n", ret
);
300 /* Fall-through (get the actual request status) */
303 ret
= viommu_get_req_errno(buf
, len
);
305 spin_unlock_irqrestore(&viommu
->request_lock
, flags
);
310 * viommu_add_mapping - add a mapping to the internal tree
312 * On success, return the new mapping. Otherwise return NULL.
314 static int viommu_add_mapping(struct viommu_domain
*vdomain
, unsigned long iova
,
315 phys_addr_t paddr
, size_t size
, u32 flags
)
317 unsigned long irqflags
;
318 struct viommu_mapping
*mapping
;
320 mapping
= kzalloc(sizeof(*mapping
), GFP_ATOMIC
);
324 mapping
->paddr
= paddr
;
325 mapping
->iova
.start
= iova
;
326 mapping
->iova
.last
= iova
+ size
- 1;
327 mapping
->flags
= flags
;
329 spin_lock_irqsave(&vdomain
->mappings_lock
, irqflags
);
330 interval_tree_insert(&mapping
->iova
, &vdomain
->mappings
);
331 spin_unlock_irqrestore(&vdomain
->mappings_lock
, irqflags
);
337 * viommu_del_mappings - remove mappings from the internal tree
339 * @vdomain: the domain
340 * @iova: start of the range
341 * @size: size of the range. A size of 0 corresponds to the entire address
344 * On success, returns the number of unmapped bytes (>= size)
346 static size_t viommu_del_mappings(struct viommu_domain
*vdomain
,
347 unsigned long iova
, size_t size
)
351 unsigned long last
= iova
+ size
- 1;
352 struct viommu_mapping
*mapping
= NULL
;
353 struct interval_tree_node
*node
, *next
;
355 spin_lock_irqsave(&vdomain
->mappings_lock
, flags
);
356 next
= interval_tree_iter_first(&vdomain
->mappings
, iova
, last
);
359 mapping
= container_of(node
, struct viommu_mapping
, iova
);
360 next
= interval_tree_iter_next(node
, iova
, last
);
362 /* Trying to split a mapping? */
363 if (mapping
->iova
.start
< iova
)
367 * Virtio-iommu doesn't allow UNMAP to split a mapping created
368 * with a single MAP request, so remove the full mapping.
370 unmapped
+= mapping
->iova
.last
- mapping
->iova
.start
+ 1;
372 interval_tree_remove(node
, &vdomain
->mappings
);
375 spin_unlock_irqrestore(&vdomain
->mappings_lock
, flags
);
381 * viommu_replay_mappings - re-send MAP requests
383 * When reattaching a domain that was previously detached from all endpoints,
384 * mappings were deleted from the device. Re-create the mappings available in
387 static int viommu_replay_mappings(struct viommu_domain
*vdomain
)
391 struct viommu_mapping
*mapping
;
392 struct interval_tree_node
*node
;
393 struct virtio_iommu_req_map map
;
395 spin_lock_irqsave(&vdomain
->mappings_lock
, flags
);
396 node
= interval_tree_iter_first(&vdomain
->mappings
, 0, -1UL);
398 mapping
= container_of(node
, struct viommu_mapping
, iova
);
399 map
= (struct virtio_iommu_req_map
) {
400 .head
.type
= VIRTIO_IOMMU_T_MAP
,
401 .domain
= cpu_to_le32(vdomain
->id
),
402 .virt_start
= cpu_to_le64(mapping
->iova
.start
),
403 .virt_end
= cpu_to_le64(mapping
->iova
.last
),
404 .phys_start
= cpu_to_le64(mapping
->paddr
),
405 .flags
= cpu_to_le32(mapping
->flags
),
408 ret
= viommu_send_req_sync(vdomain
->viommu
, &map
, sizeof(map
));
412 node
= interval_tree_iter_next(node
, 0, -1UL);
414 spin_unlock_irqrestore(&vdomain
->mappings_lock
, flags
);
419 static int viommu_add_resv_mem(struct viommu_endpoint
*vdev
,
420 struct virtio_iommu_probe_resv_mem
*mem
,
425 phys_addr_t start
, end
;
426 struct iommu_resv_region
*region
= NULL
;
427 unsigned long prot
= IOMMU_WRITE
| IOMMU_NOEXEC
| IOMMU_MMIO
;
429 start
= start64
= le64_to_cpu(mem
->start
);
430 end
= end64
= le64_to_cpu(mem
->end
);
431 size
= end64
- start64
+ 1;
433 /* Catch any overflow, including the unlikely end64 - start64 + 1 = 0 */
434 if (start
!= start64
|| end
!= end64
|| size
< end64
- start64
)
437 if (len
< sizeof(*mem
))
440 switch (mem
->subtype
) {
442 dev_warn(vdev
->dev
, "unknown resv mem subtype 0x%x\n",
445 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED
:
446 region
= iommu_alloc_resv_region(start
, size
, 0,
447 IOMMU_RESV_RESERVED
);
449 case VIRTIO_IOMMU_RESV_MEM_T_MSI
:
450 region
= iommu_alloc_resv_region(start
, size
, prot
,
457 list_add(&vdev
->resv_regions
, ®ion
->list
);
461 static int viommu_probe_endpoint(struct viommu_dev
*viommu
, struct device
*dev
)
467 struct virtio_iommu_req_probe
*probe
;
468 struct virtio_iommu_probe_property
*prop
;
469 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
470 struct viommu_endpoint
*vdev
= fwspec
->iommu_priv
;
472 if (!fwspec
->num_ids
)
475 probe_len
= sizeof(*probe
) + viommu
->probe_size
+
476 sizeof(struct virtio_iommu_req_tail
);
477 probe
= kzalloc(probe_len
, GFP_KERNEL
);
481 probe
->head
.type
= VIRTIO_IOMMU_T_PROBE
;
483 * For now, assume that properties of an endpoint that outputs multiple
484 * IDs are consistent. Only probe the first one.
486 probe
->endpoint
= cpu_to_le32(fwspec
->ids
[0]);
488 ret
= viommu_send_req_sync(viommu
, probe
, probe_len
);
492 prop
= (void *)probe
->properties
;
493 type
= le16_to_cpu(prop
->type
) & VIRTIO_IOMMU_PROBE_T_MASK
;
495 while (type
!= VIRTIO_IOMMU_PROBE_T_NONE
&&
496 cur
< viommu
->probe_size
) {
497 len
= le16_to_cpu(prop
->length
) + sizeof(*prop
);
500 case VIRTIO_IOMMU_PROBE_T_RESV_MEM
:
501 ret
= viommu_add_resv_mem(vdev
, (void *)prop
, len
);
504 dev_err(dev
, "unknown viommu prop 0x%x\n", type
);
508 dev_err(dev
, "failed to parse viommu prop 0x%x\n", type
);
511 if (cur
>= viommu
->probe_size
)
514 prop
= (void *)probe
->properties
+ cur
;
515 type
= le16_to_cpu(prop
->type
) & VIRTIO_IOMMU_PROBE_T_MASK
;
523 static int viommu_fault_handler(struct viommu_dev
*viommu
,
524 struct virtio_iommu_fault
*fault
)
528 u8 reason
= fault
->reason
;
529 u32 flags
= le32_to_cpu(fault
->flags
);
530 u32 endpoint
= le32_to_cpu(fault
->endpoint
);
531 u64 address
= le64_to_cpu(fault
->address
);
534 case VIRTIO_IOMMU_FAULT_R_DOMAIN
:
535 reason_str
= "domain";
537 case VIRTIO_IOMMU_FAULT_R_MAPPING
:
540 case VIRTIO_IOMMU_FAULT_R_UNKNOWN
:
542 reason_str
= "unknown";
546 /* TODO: find EP by ID and report_iommu_fault */
547 if (flags
& VIRTIO_IOMMU_FAULT_F_ADDRESS
)
548 dev_err_ratelimited(viommu
->dev
, "%s fault from EP %u at %#llx [%s%s%s]\n",
549 reason_str
, endpoint
, address
,
550 flags
& VIRTIO_IOMMU_FAULT_F_READ
? "R" : "",
551 flags
& VIRTIO_IOMMU_FAULT_F_WRITE
? "W" : "",
552 flags
& VIRTIO_IOMMU_FAULT_F_EXEC
? "X" : "");
554 dev_err_ratelimited(viommu
->dev
, "%s fault from EP %u\n",
555 reason_str
, endpoint
);
559 static void viommu_event_handler(struct virtqueue
*vq
)
563 struct scatterlist sg
[1];
564 struct viommu_event
*evt
;
565 struct viommu_dev
*viommu
= vq
->vdev
->priv
;
567 while ((evt
= virtqueue_get_buf(vq
, &len
)) != NULL
) {
568 if (len
> sizeof(*evt
)) {
570 "invalid event buffer (len %u != %zu)\n",
572 } else if (!(evt
->head
& VIOMMU_FAULT_RESV_MASK
)) {
573 viommu_fault_handler(viommu
, &evt
->fault
);
576 sg_init_one(sg
, evt
, sizeof(*evt
));
577 ret
= virtqueue_add_inbuf(vq
, sg
, 1, evt
, GFP_ATOMIC
);
579 dev_err(viommu
->dev
, "could not add event buffer\n");
587 static struct iommu_domain
*viommu_domain_alloc(unsigned type
)
589 struct viommu_domain
*vdomain
;
591 if (type
!= IOMMU_DOMAIN_UNMANAGED
&& type
!= IOMMU_DOMAIN_DMA
)
594 vdomain
= kzalloc(sizeof(*vdomain
), GFP_KERNEL
);
598 mutex_init(&vdomain
->mutex
);
599 spin_lock_init(&vdomain
->mappings_lock
);
600 vdomain
->mappings
= RB_ROOT_CACHED
;
602 if (type
== IOMMU_DOMAIN_DMA
&&
603 iommu_get_dma_cookie(&vdomain
->domain
)) {
608 return &vdomain
->domain
;
611 static int viommu_domain_finalise(struct viommu_dev
*viommu
,
612 struct iommu_domain
*domain
)
615 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
617 vdomain
->viommu
= viommu
;
618 vdomain
->map_flags
= viommu
->map_flags
;
620 domain
->pgsize_bitmap
= viommu
->pgsize_bitmap
;
621 domain
->geometry
= viommu
->geometry
;
623 ret
= ida_alloc_range(&viommu
->domain_ids
, viommu
->first_domain
,
624 viommu
->last_domain
, GFP_KERNEL
);
626 vdomain
->id
= (unsigned int)ret
;
628 return ret
> 0 ? 0 : ret
;
631 static void viommu_domain_free(struct iommu_domain
*domain
)
633 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
635 iommu_put_dma_cookie(domain
);
637 /* Free all remaining mappings (size 2^64) */
638 viommu_del_mappings(vdomain
, 0, 0);
641 ida_free(&vdomain
->viommu
->domain_ids
, vdomain
->id
);
646 static int viommu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
650 struct virtio_iommu_req_attach req
;
651 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
652 struct viommu_endpoint
*vdev
= fwspec
->iommu_priv
;
653 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
655 mutex_lock(&vdomain
->mutex
);
656 if (!vdomain
->viommu
) {
658 * Properly initialize the domain now that we know which viommu
661 ret
= viommu_domain_finalise(vdev
->viommu
, domain
);
662 } else if (vdomain
->viommu
!= vdev
->viommu
) {
663 dev_err(dev
, "cannot attach to foreign vIOMMU\n");
666 mutex_unlock(&vdomain
->mutex
);
672 * In the virtio-iommu device, when attaching the endpoint to a new
673 * domain, it is detached from the old one and, if as as a result the
674 * old domain isn't attached to any endpoint, all mappings are removed
675 * from the old domain and it is freed.
677 * In the driver the old domain still exists, and its mappings will be
678 * recreated if it gets reattached to an endpoint. Otherwise it will be
681 * vdev->vdomain is protected by group->mutex
684 vdev
->vdomain
->nr_endpoints
--;
686 req
= (struct virtio_iommu_req_attach
) {
687 .head
.type
= VIRTIO_IOMMU_T_ATTACH
,
688 .domain
= cpu_to_le32(vdomain
->id
),
691 for (i
= 0; i
< fwspec
->num_ids
; i
++) {
692 req
.endpoint
= cpu_to_le32(fwspec
->ids
[i
]);
694 ret
= viommu_send_req_sync(vdomain
->viommu
, &req
, sizeof(req
));
699 if (!vdomain
->nr_endpoints
) {
701 * This endpoint is the first to be attached to the domain.
702 * Replay existing mappings (e.g. SW MSI).
704 ret
= viommu_replay_mappings(vdomain
);
709 vdomain
->nr_endpoints
++;
710 vdev
->vdomain
= vdomain
;
715 static int viommu_map(struct iommu_domain
*domain
, unsigned long iova
,
716 phys_addr_t paddr
, size_t size
, int prot
)
720 struct virtio_iommu_req_map map
;
721 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
723 flags
= (prot
& IOMMU_READ
? VIRTIO_IOMMU_MAP_F_READ
: 0) |
724 (prot
& IOMMU_WRITE
? VIRTIO_IOMMU_MAP_F_WRITE
: 0) |
725 (prot
& IOMMU_MMIO
? VIRTIO_IOMMU_MAP_F_MMIO
: 0);
727 if (flags
& ~vdomain
->map_flags
)
730 ret
= viommu_add_mapping(vdomain
, iova
, paddr
, size
, flags
);
734 map
= (struct virtio_iommu_req_map
) {
735 .head
.type
= VIRTIO_IOMMU_T_MAP
,
736 .domain
= cpu_to_le32(vdomain
->id
),
737 .virt_start
= cpu_to_le64(iova
),
738 .phys_start
= cpu_to_le64(paddr
),
739 .virt_end
= cpu_to_le64(iova
+ size
- 1),
740 .flags
= cpu_to_le32(flags
),
743 if (!vdomain
->nr_endpoints
)
746 ret
= viommu_send_req_sync(vdomain
->viommu
, &map
, sizeof(map
));
748 viommu_del_mappings(vdomain
, iova
, size
);
753 static size_t viommu_unmap(struct iommu_domain
*domain
, unsigned long iova
,
754 size_t size
, struct iommu_iotlb_gather
*gather
)
758 struct virtio_iommu_req_unmap unmap
;
759 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
761 unmapped
= viommu_del_mappings(vdomain
, iova
, size
);
765 /* Device already removed all mappings after detach. */
766 if (!vdomain
->nr_endpoints
)
769 unmap
= (struct virtio_iommu_req_unmap
) {
770 .head
.type
= VIRTIO_IOMMU_T_UNMAP
,
771 .domain
= cpu_to_le32(vdomain
->id
),
772 .virt_start
= cpu_to_le64(iova
),
773 .virt_end
= cpu_to_le64(iova
+ unmapped
- 1),
776 ret
= viommu_add_req(vdomain
->viommu
, &unmap
, sizeof(unmap
));
777 return ret
? 0 : unmapped
;
780 static phys_addr_t
viommu_iova_to_phys(struct iommu_domain
*domain
,
785 struct viommu_mapping
*mapping
;
786 struct interval_tree_node
*node
;
787 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
789 spin_lock_irqsave(&vdomain
->mappings_lock
, flags
);
790 node
= interval_tree_iter_first(&vdomain
->mappings
, iova
, iova
);
792 mapping
= container_of(node
, struct viommu_mapping
, iova
);
793 paddr
= mapping
->paddr
+ (iova
- mapping
->iova
.start
);
795 spin_unlock_irqrestore(&vdomain
->mappings_lock
, flags
);
800 static void viommu_iotlb_sync(struct iommu_domain
*domain
,
801 struct iommu_iotlb_gather
*gather
)
803 struct viommu_domain
*vdomain
= to_viommu_domain(domain
);
805 viommu_sync_req(vdomain
->viommu
);
808 static void viommu_get_resv_regions(struct device
*dev
, struct list_head
*head
)
810 struct iommu_resv_region
*entry
, *new_entry
, *msi
= NULL
;
811 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
812 struct viommu_endpoint
*vdev
= fwspec
->iommu_priv
;
813 int prot
= IOMMU_WRITE
| IOMMU_NOEXEC
| IOMMU_MMIO
;
815 list_for_each_entry(entry
, &vdev
->resv_regions
, list
) {
816 if (entry
->type
== IOMMU_RESV_MSI
)
819 new_entry
= kmemdup(entry
, sizeof(*entry
), GFP_KERNEL
);
822 list_add_tail(&new_entry
->list
, head
);
826 * If the device didn't register any bypass MSI window, add a
827 * software-mapped region.
830 msi
= iommu_alloc_resv_region(MSI_IOVA_BASE
, MSI_IOVA_LENGTH
,
831 prot
, IOMMU_RESV_SW_MSI
);
835 list_add_tail(&msi
->list
, head
);
838 iommu_dma_get_resv_regions(dev
, head
);
841 static void viommu_put_resv_regions(struct device
*dev
, struct list_head
*head
)
843 struct iommu_resv_region
*entry
, *next
;
845 list_for_each_entry_safe(entry
, next
, head
, list
)
849 static struct iommu_ops viommu_ops
;
850 static struct virtio_driver virtio_iommu_drv
;
852 static int viommu_match_node(struct device
*dev
, const void *data
)
854 return dev
->parent
->fwnode
== data
;
857 static struct viommu_dev
*viommu_get_by_fwnode(struct fwnode_handle
*fwnode
)
859 struct device
*dev
= driver_find_device(&virtio_iommu_drv
.driver
, NULL
,
860 fwnode
, viommu_match_node
);
863 return dev
? dev_to_virtio(dev
)->priv
: NULL
;
866 static int viommu_add_device(struct device
*dev
)
869 struct iommu_group
*group
;
870 struct viommu_endpoint
*vdev
;
871 struct viommu_dev
*viommu
= NULL
;
872 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
874 if (!fwspec
|| fwspec
->ops
!= &viommu_ops
)
877 viommu
= viommu_get_by_fwnode(fwspec
->iommu_fwnode
);
881 vdev
= kzalloc(sizeof(*vdev
), GFP_KERNEL
);
886 vdev
->viommu
= viommu
;
887 INIT_LIST_HEAD(&vdev
->resv_regions
);
888 fwspec
->iommu_priv
= vdev
;
890 if (viommu
->probe_size
) {
891 /* Get additional information for this endpoint */
892 ret
= viommu_probe_endpoint(viommu
, dev
);
897 ret
= iommu_device_link(&viommu
->iommu
, dev
);
902 * Last step creates a default domain and attaches to it. Everything
905 group
= iommu_group_get_for_dev(dev
);
907 ret
= PTR_ERR(group
);
911 iommu_group_put(group
);
913 return PTR_ERR_OR_ZERO(group
);
916 iommu_device_unlink(&viommu
->iommu
, dev
);
918 viommu_put_resv_regions(dev
, &vdev
->resv_regions
);
924 static void viommu_remove_device(struct device
*dev
)
926 struct viommu_endpoint
*vdev
;
927 struct iommu_fwspec
*fwspec
= dev_iommu_fwspec_get(dev
);
929 if (!fwspec
|| fwspec
->ops
!= &viommu_ops
)
932 vdev
= fwspec
->iommu_priv
;
934 iommu_group_remove_device(dev
);
935 iommu_device_unlink(&vdev
->viommu
->iommu
, dev
);
936 viommu_put_resv_regions(dev
, &vdev
->resv_regions
);
940 static struct iommu_group
*viommu_device_group(struct device
*dev
)
943 return pci_device_group(dev
);
945 return generic_device_group(dev
);
948 static int viommu_of_xlate(struct device
*dev
, struct of_phandle_args
*args
)
950 return iommu_fwspec_add_ids(dev
, args
->args
, 1);
953 static struct iommu_ops viommu_ops
= {
954 .domain_alloc
= viommu_domain_alloc
,
955 .domain_free
= viommu_domain_free
,
956 .attach_dev
= viommu_attach_dev
,
958 .unmap
= viommu_unmap
,
959 .iova_to_phys
= viommu_iova_to_phys
,
960 .iotlb_sync
= viommu_iotlb_sync
,
961 .add_device
= viommu_add_device
,
962 .remove_device
= viommu_remove_device
,
963 .device_group
= viommu_device_group
,
964 .get_resv_regions
= viommu_get_resv_regions
,
965 .put_resv_regions
= viommu_put_resv_regions
,
966 .of_xlate
= viommu_of_xlate
,
969 static int viommu_init_vqs(struct viommu_dev
*viommu
)
971 struct virtio_device
*vdev
= dev_to_virtio(viommu
->dev
);
972 const char *names
[] = { "request", "event" };
973 vq_callback_t
*callbacks
[] = {
974 NULL
, /* No async requests */
975 viommu_event_handler
,
978 return virtio_find_vqs(vdev
, VIOMMU_NR_VQS
, viommu
->vqs
, callbacks
,
982 static int viommu_fill_evtq(struct viommu_dev
*viommu
)
985 struct scatterlist sg
[1];
986 struct viommu_event
*evts
;
987 struct virtqueue
*vq
= viommu
->vqs
[VIOMMU_EVENT_VQ
];
988 size_t nr_evts
= vq
->num_free
;
990 viommu
->evts
= evts
= devm_kmalloc_array(viommu
->dev
, nr_evts
,
991 sizeof(*evts
), GFP_KERNEL
);
995 for (i
= 0; i
< nr_evts
; i
++) {
996 sg_init_one(sg
, &evts
[i
], sizeof(*evts
));
997 ret
= virtqueue_add_inbuf(vq
, sg
, 1, &evts
[i
], GFP_KERNEL
);
1005 static int viommu_probe(struct virtio_device
*vdev
)
1007 struct device
*parent_dev
= vdev
->dev
.parent
;
1008 struct viommu_dev
*viommu
= NULL
;
1009 struct device
*dev
= &vdev
->dev
;
1010 u64 input_start
= 0;
1011 u64 input_end
= -1UL;
1014 if (!virtio_has_feature(vdev
, VIRTIO_F_VERSION_1
) ||
1015 !virtio_has_feature(vdev
, VIRTIO_IOMMU_F_MAP_UNMAP
))
1018 viommu
= devm_kzalloc(dev
, sizeof(*viommu
), GFP_KERNEL
);
1022 spin_lock_init(&viommu
->request_lock
);
1023 ida_init(&viommu
->domain_ids
);
1025 viommu
->vdev
= vdev
;
1026 INIT_LIST_HEAD(&viommu
->requests
);
1028 ret
= viommu_init_vqs(viommu
);
1032 virtio_cread(vdev
, struct virtio_iommu_config
, page_size_mask
,
1033 &viommu
->pgsize_bitmap
);
1035 if (!viommu
->pgsize_bitmap
) {
1040 viommu
->map_flags
= VIRTIO_IOMMU_MAP_F_READ
| VIRTIO_IOMMU_MAP_F_WRITE
;
1041 viommu
->last_domain
= ~0U;
1043 /* Optional features */
1044 virtio_cread_feature(vdev
, VIRTIO_IOMMU_F_INPUT_RANGE
,
1045 struct virtio_iommu_config
, input_range
.start
,
1048 virtio_cread_feature(vdev
, VIRTIO_IOMMU_F_INPUT_RANGE
,
1049 struct virtio_iommu_config
, input_range
.end
,
1052 virtio_cread_feature(vdev
, VIRTIO_IOMMU_F_DOMAIN_RANGE
,
1053 struct virtio_iommu_config
, domain_range
.start
,
1054 &viommu
->first_domain
);
1056 virtio_cread_feature(vdev
, VIRTIO_IOMMU_F_DOMAIN_RANGE
,
1057 struct virtio_iommu_config
, domain_range
.end
,
1058 &viommu
->last_domain
);
1060 virtio_cread_feature(vdev
, VIRTIO_IOMMU_F_PROBE
,
1061 struct virtio_iommu_config
, probe_size
,
1062 &viommu
->probe_size
);
1064 viommu
->geometry
= (struct iommu_domain_geometry
) {
1065 .aperture_start
= input_start
,
1066 .aperture_end
= input_end
,
1067 .force_aperture
= true,
1070 if (virtio_has_feature(vdev
, VIRTIO_IOMMU_F_MMIO
))
1071 viommu
->map_flags
|= VIRTIO_IOMMU_MAP_F_MMIO
;
1073 viommu_ops
.pgsize_bitmap
= viommu
->pgsize_bitmap
;
1075 virtio_device_ready(vdev
);
1077 /* Populate the event queue with buffers */
1078 ret
= viommu_fill_evtq(viommu
);
1082 ret
= iommu_device_sysfs_add(&viommu
->iommu
, dev
, NULL
, "%s",
1083 virtio_bus_name(vdev
));
1087 iommu_device_set_ops(&viommu
->iommu
, &viommu_ops
);
1088 iommu_device_set_fwnode(&viommu
->iommu
, parent_dev
->fwnode
);
1090 iommu_device_register(&viommu
->iommu
);
1093 if (pci_bus_type
.iommu_ops
!= &viommu_ops
) {
1095 ret
= bus_set_iommu(&pci_bus_type
, &viommu_ops
);
1097 goto err_unregister
;
1100 #ifdef CONFIG_ARM_AMBA
1101 if (amba_bustype
.iommu_ops
!= &viommu_ops
) {
1102 ret
= bus_set_iommu(&amba_bustype
, &viommu_ops
);
1104 goto err_unregister
;
1107 if (platform_bus_type
.iommu_ops
!= &viommu_ops
) {
1108 ret
= bus_set_iommu(&platform_bus_type
, &viommu_ops
);
1110 goto err_unregister
;
1113 vdev
->priv
= viommu
;
1115 dev_info(dev
, "input address: %u bits\n",
1116 order_base_2(viommu
->geometry
.aperture_end
));
1117 dev_info(dev
, "page mask: %#llx\n", viommu
->pgsize_bitmap
);
1122 iommu_device_sysfs_remove(&viommu
->iommu
);
1123 iommu_device_unregister(&viommu
->iommu
);
1125 vdev
->config
->del_vqs(vdev
);
1130 static void viommu_remove(struct virtio_device
*vdev
)
1132 struct viommu_dev
*viommu
= vdev
->priv
;
1134 iommu_device_sysfs_remove(&viommu
->iommu
);
1135 iommu_device_unregister(&viommu
->iommu
);
1137 /* Stop all virtqueues */
1138 vdev
->config
->reset(vdev
);
1139 vdev
->config
->del_vqs(vdev
);
1141 dev_info(&vdev
->dev
, "device removed\n");
1144 static void viommu_config_changed(struct virtio_device
*vdev
)
1146 dev_warn(&vdev
->dev
, "config changed\n");
1149 static unsigned int features
[] = {
1150 VIRTIO_IOMMU_F_MAP_UNMAP
,
1151 VIRTIO_IOMMU_F_INPUT_RANGE
,
1152 VIRTIO_IOMMU_F_DOMAIN_RANGE
,
1153 VIRTIO_IOMMU_F_PROBE
,
1154 VIRTIO_IOMMU_F_MMIO
,
1157 static struct virtio_device_id id_table
[] = {
1158 { VIRTIO_ID_IOMMU
, VIRTIO_DEV_ANY_ID
},
1162 static struct virtio_driver virtio_iommu_drv
= {
1163 .driver
.name
= KBUILD_MODNAME
,
1164 .driver
.owner
= THIS_MODULE
,
1165 .id_table
= id_table
,
1166 .feature_table
= features
,
1167 .feature_table_size
= ARRAY_SIZE(features
),
1168 .probe
= viommu_probe
,
1169 .remove
= viommu_remove
,
1170 .config_changed
= viommu_config_changed
,
1173 module_virtio_driver(virtio_iommu_drv
);
1175 MODULE_DESCRIPTION("Virtio IOMMU driver");
1176 MODULE_AUTHOR("Jean-Philippe Brucker <jean-philippe.brucker@arm.com>");
1177 MODULE_LICENSE("GPL v2");