drivers/iommu/iommufd/fault.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2024 Intel Corporation
   3  */
   4 #define pr_fmt(fmt) "iommufd: " fmt
   5
   6 #include <linux/anon_inodes.h>
   7 #include <linux/file.h>
   8 #include <linux/fs.h>
   9 #include <linux/iommufd.h>
  10 #include <linux/module.h>
  11 #include <linux/mutex.h>
  12 #include <linux/pci.h>
  13 #include <linux/pci-ats.h>
  14 #include <linux/poll.h>
  15 #include <uapi/linux/iommufd.h>
  16
  17 #include "../iommu-priv.h"
  18 #include "iommufd_private.h"
  19
  20 static int iommufd_fault_iopf_enable(struct iommufd_device *idev)
  21 {
  22         struct device *dev = idev->dev;
  23         int ret;
  24
  25         /*
  26          * Once we turn on PCI/PRI support for VF, the response failure code
  27          * should not be forwarded to the hardware due to PRI being a shared
  28          * resource between PF and VFs. There is no coordination for this
  29          * shared capability. This waits for a vPRI reset to recover.
  30          */
  31         if (dev_is_pci(dev)) {
  32                 struct pci_dev *pdev = to_pci_dev(dev);
  33
  34                 if (pdev->is_virtfn && pci_pri_supported(pdev))
  35                         return -EINVAL;
  36         }
  37
  38         mutex_lock(&idev->iopf_lock);
  39         /* Device iopf has already been on. */
  40         if (++idev->iopf_enabled > 1) {
  41                 mutex_unlock(&idev->iopf_lock);
  42                 return 0;
  43         }
  44
  45         ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF);
  46         if (ret)
  47                 --idev->iopf_enabled;
  48         mutex_unlock(&idev->iopf_lock);
  49
  50         return ret;
  51 }
  52
  53 static void iommufd_fault_iopf_disable(struct iommufd_device *idev)
  54 {
  55         mutex_lock(&idev->iopf_lock);
  56         if (!WARN_ON(idev->iopf_enabled == 0)) {
  57                 if (--idev->iopf_enabled == 0)
  58                         iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF);
  59         }
  60         mutex_unlock(&idev->iopf_lock);
  61 }
  62
  63 static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
  64                                      struct iommufd_device *idev)
  65 {
  66         struct iommufd_attach_handle *handle;
  67         int ret;
  68
  69         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
  70         if (!handle)
  71                 return -ENOMEM;
  72
  73         handle->idev = idev;
  74         ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group,
  75                                         &handle->handle);
  76         if (ret)
  77                 kfree(handle);
  78
  79         return ret;
  80 }
  81
  82 int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt,
  83                                     struct iommufd_device *idev)
  84 {
  85         int ret;
  86
  87         if (!hwpt->fault)
  88                 return -EINVAL;
  89
  90         ret = iommufd_fault_iopf_enable(idev);
  91         if (ret)
  92                 return ret;
  93
  94         ret = __fault_domain_attach_dev(hwpt, idev);
  95         if (ret)
  96                 iommufd_fault_iopf_disable(idev);
  97
  98         return ret;
  99 }
 100
 101 static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt,
 102                                          struct iommufd_attach_handle *handle)
 103 {
 104         struct iommufd_fault *fault = hwpt->fault;
 105         struct iopf_group *group, *next;
 106         unsigned long index;
 107
 108         if (!fault)
 109                 return;
 110
 111         mutex_lock(&fault->mutex);
 112         list_for_each_entry_safe(group, next, &fault->deliver, node) {
 113                 if (group->attach_handle != &handle->handle)
 114                         continue;
 115                 list_del(&group->node);
 116                 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
 117                 iopf_free_group(group);
 118         }
 119
 120         xa_for_each(&fault->response, index, group) {
 121                 if (group->attach_handle != &handle->handle)
 122                         continue;
 123                 xa_erase(&fault->response, index);
 124                 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
 125                 iopf_free_group(group);
 126         }
 127         mutex_unlock(&fault->mutex);
 128 }
 129
 130 static struct iommufd_attach_handle *
 131 iommufd_device_get_attach_handle(struct iommufd_device *idev)
 132 {
 133         struct iommu_attach_handle *handle;
 134
 135         handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0);
 136         if (IS_ERR(handle))
 137                 return NULL;
 138
 139         return to_iommufd_handle(handle);
 140 }
 141
 142 void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt,
 143                                      struct iommufd_device *idev)
 144 {
 145         struct iommufd_attach_handle *handle;
 146
 147         handle = iommufd_device_get_attach_handle(idev);
 148         iommu_detach_group_handle(hwpt->domain, idev->igroup->group);
 149         iommufd_auto_response_faults(hwpt, handle);
 150         iommufd_fault_iopf_disable(idev);
 151         kfree(handle);
 152 }
 153
 154 static int __fault_domain_replace_dev(struct iommufd_device *idev,
 155                                       struct iommufd_hw_pagetable *hwpt,
 156                                       struct iommufd_hw_pagetable *old)
 157 {
 158         struct iommufd_attach_handle *handle, *curr = NULL;
 159         int ret;
 160
 161         if (old->fault)
 162                 curr = iommufd_device_get_attach_handle(idev);
 163
 164         if (hwpt->fault) {
 165                 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 166                 if (!handle)
 167                         return -ENOMEM;
 168
 169                 handle->idev = idev;
 170                 ret = iommu_replace_group_handle(idev->igroup->group,
 171                                                  hwpt->domain, &handle->handle);
 172         } else {
 173                 ret = iommu_replace_group_handle(idev->igroup->group,
 174                                                  hwpt->domain, NULL);
 175         }
 176
 177         if (!ret && curr) {
 178                 iommufd_auto_response_faults(old, curr);
 179                 kfree(curr);
 180         }
 181
 182         return ret;
 183 }
 184
 185 int iommufd_fault_domain_replace_dev(struct iommufd_device *idev,
 186                                      struct iommufd_hw_pagetable *hwpt,
 187                                      struct iommufd_hw_pagetable *old)
 188 {
 189         bool iopf_off = !hwpt->fault && old->fault;
 190         bool iopf_on = hwpt->fault && !old->fault;
 191         int ret;
 192
 193         if (iopf_on) {
 194                 ret = iommufd_fault_iopf_enable(idev);
 195                 if (ret)
 196                         return ret;
 197         }
 198
 199         ret = __fault_domain_replace_dev(idev, hwpt, old);
 200         if (ret) {
 201                 if (iopf_on)
 202                         iommufd_fault_iopf_disable(idev);
 203                 return ret;
 204         }
 205
 206         if (iopf_off)
 207                 iommufd_fault_iopf_disable(idev);
 208
 209         return 0;
 210 }
 211
 212 void iommufd_fault_destroy(struct iommufd_object *obj)
 213 {
 214         struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
 215         struct iopf_group *group, *next;
 216
 217         /*
 218          * The iommufd object's reference count is zero at this point.
 219          * We can be confident that no other threads are currently
 220          * accessing this pointer. Therefore, acquiring the mutex here
 221          * is unnecessary.
 222          */
 223         list_for_each_entry_safe(group, next, &fault->deliver, node) {
 224                 list_del(&group->node);
 225                 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
 226                 iopf_free_group(group);
 227         }
 228 }
 229
 230 static void iommufd_compose_fault_message(struct iommu_fault *fault,
 231                                           struct iommu_hwpt_pgfault *hwpt_fault,
 232                                           struct iommufd_device *idev,
 233                                           u32 cookie)
 234 {
 235         hwpt_fault->flags = fault->prm.flags;
 236         hwpt_fault->dev_id = idev->obj.id;
 237         hwpt_fault->pasid = fault->prm.pasid;
 238         hwpt_fault->grpid = fault->prm.grpid;
 239         hwpt_fault->perm = fault->prm.perm;
 240         hwpt_fault->addr = fault->prm.addr;
 241         hwpt_fault->length = 0;
 242         hwpt_fault->cookie = cookie;
 243 }
 244
 245 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
 246                                        size_t count, loff_t *ppos)
 247 {
 248         size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
 249         struct iommufd_fault *fault = filep->private_data;
 250         struct iommu_hwpt_pgfault data;
 251         struct iommufd_device *idev;
 252         struct iopf_group *group;
 253         struct iopf_fault *iopf;
 254         size_t done = 0;
 255         int rc = 0;
 256
 257         if (*ppos || count % fault_size)
 258                 return -ESPIPE;
 259
 260         mutex_lock(&fault->mutex);
 261         while (!list_empty(&fault->deliver) && count > done) {
 262                 group = list_first_entry(&fault->deliver,
 263                                          struct iopf_group, node);
 264
 265                 if (group->fault_count * fault_size > count - done)
 266                         break;
 267
 268                 rc = xa_alloc(&fault->response, &group->cookie, group,
 269                               xa_limit_32b, GFP_KERNEL);
 270                 if (rc)
 271                         break;
 272
 273                 idev = to_iommufd_handle(group->attach_handle)->idev;
 274                 list_for_each_entry(iopf, &group->faults, list) {
 275                         iommufd_compose_fault_message(&iopf->fault,
 276                                                       &data, idev,
 277                                                       group->cookie);
 278                         if (copy_to_user(buf + done, &data, fault_size)) {
 279                                 xa_erase(&fault->response, group->cookie);
 280                                 rc = -EFAULT;
 281                                 break;
 282                         }
 283                         done += fault_size;
 284                 }
 285
 286                 list_del(&group->node);
 287         }
 288         mutex_unlock(&fault->mutex);
 289
 290         return done == 0 ? rc : done;
 291 }
 292
 293 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
 294                                         size_t count, loff_t *ppos)
 295 {
 296         size_t response_size = sizeof(struct iommu_hwpt_page_response);
 297         struct iommufd_fault *fault = filep->private_data;
 298         struct iommu_hwpt_page_response response;
 299         struct iopf_group *group;
 300         size_t done = 0;
 301         int rc = 0;
 302
 303         if (*ppos || count % response_size)
 304                 return -ESPIPE;
 305
 306         mutex_lock(&fault->mutex);
 307         while (count > done) {
 308                 rc = copy_from_user(&response, buf + done, response_size);
 309                 if (rc)
 310                         break;
 311
 312                 static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS ==
 313                               (int)IOMMU_PAGE_RESP_SUCCESS);
 314                 static_assert((int)IOMMUFD_PAGE_RESP_INVALID ==
 315                               (int)IOMMU_PAGE_RESP_INVALID);
 316                 if (response.code != IOMMUFD_PAGE_RESP_SUCCESS &&
 317                     response.code != IOMMUFD_PAGE_RESP_INVALID) {
 318                         rc = -EINVAL;
 319                         break;
 320                 }
 321
 322                 group = xa_erase(&fault->response, response.cookie);
 323                 if (!group) {
 324                         rc = -EINVAL;
 325                         break;
 326                 }
 327
 328                 iopf_group_response(group, response.code);
 329                 iopf_free_group(group);
 330                 done += response_size;
 331         }
 332         mutex_unlock(&fault->mutex);
 333
 334         return done == 0 ? rc : done;
 335 }
 336
 337 static __poll_t iommufd_fault_fops_poll(struct file *filep,
 338                                         struct poll_table_struct *wait)
 339 {
 340         struct iommufd_fault *fault = filep->private_data;
 341         __poll_t pollflags = EPOLLOUT;
 342
 343         poll_wait(filep, &fault->wait_queue, wait);
 344         mutex_lock(&fault->mutex);
 345         if (!list_empty(&fault->deliver))
 346                 pollflags |= EPOLLIN | EPOLLRDNORM;
 347         mutex_unlock(&fault->mutex);
 348
 349         return pollflags;
 350 }
 351
 352 static int iommufd_fault_fops_release(struct inode *inode, struct file *filep)
 353 {
 354         struct iommufd_fault *fault = filep->private_data;
 355
 356         refcount_dec(&fault->obj.users);
 357         iommufd_ctx_put(fault->ictx);
 358         return 0;
 359 }
 360
 361 static const struct file_operations iommufd_fault_fops = {
 362         .owner          = THIS_MODULE,
 363         .open           = nonseekable_open,
 364         .read           = iommufd_fault_fops_read,
 365         .write          = iommufd_fault_fops_write,
 366         .poll           = iommufd_fault_fops_poll,
 367         .release        = iommufd_fault_fops_release,
 368 };
 369
 370 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
 371 {
 372         struct iommu_fault_alloc *cmd = ucmd->cmd;
 373         struct iommufd_fault *fault;
 374         struct file *filep;
 375         int fdno;
 376         int rc;
 377
 378         if (cmd->flags)
 379                 return -EOPNOTSUPP;
 380
 381         fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
 382         if (IS_ERR(fault))
 383                 return PTR_ERR(fault);
 384
 385         fault->ictx = ucmd->ictx;
 386         INIT_LIST_HEAD(&fault->deliver);
 387         xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
 388         mutex_init(&fault->mutex);
 389         init_waitqueue_head(&fault->wait_queue);
 390
 391         filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
 392                                    fault, O_RDWR);
 393         if (IS_ERR(filep)) {
 394                 rc = PTR_ERR(filep);
 395                 goto out_abort;
 396         }
 397
 398         refcount_inc(&fault->obj.users);
 399         iommufd_ctx_get(fault->ictx);
 400         fault->filep = filep;
 401
 402         fdno = get_unused_fd_flags(O_CLOEXEC);
 403         if (fdno < 0) {
 404                 rc = fdno;
 405                 goto out_fput;
 406         }
 407
 408         cmd->out_fault_id = fault->obj.id;
 409         cmd->out_fault_fd = fdno;
 410
 411         rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
 412         if (rc)
 413                 goto out_put_fdno;
 414         iommufd_object_finalize(ucmd->ictx, &fault->obj);
 415
 416         fd_install(fdno, fault->filep);
 417
 418         return 0;
 419 out_put_fdno:
 420         put_unused_fd(fdno);
 421 out_fput:
 422         fput(filep);
 423         refcount_dec(&fault->obj.users);
 424         iommufd_ctx_put(fault->ictx);
 425 out_abort:
 426         iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
 427
 428         return rc;
 429 }
 430
 431 int iommufd_fault_iopf_handler(struct iopf_group *group)
 432 {
 433         struct iommufd_hw_pagetable *hwpt;
 434         struct iommufd_fault *fault;
 435
 436         hwpt = group->attach_handle->domain->fault_data;
 437         fault = hwpt->fault;
 438
 439         mutex_lock(&fault->mutex);
 440         list_add_tail(&group->node, &fault->deliver);
 441         mutex_unlock(&fault->mutex);
 442
 443         wake_up_interruptible(&fault->wait_queue);
 444
 445         return 0;
 446 }