drivers/vfio/pci/pds/lm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2023 Advanced Micro Devices, Inc. */
   3
   4 #include <linux/anon_inodes.h>
   5 #include <linux/file.h>
   6 #include <linux/fs.h>
   7 #include <linux/highmem.h>
   8 #include <linux/vfio.h>
   9 #include <linux/vfio_pci_core.h>
  10
  11 #include "vfio_dev.h"
  12 #include "cmds.h"
  13
  14 static struct pds_vfio_lm_file *
  15 pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size)
  16 {
  17         struct pds_vfio_lm_file *lm_file = NULL;
  18         unsigned long long npages;
  19         struct page **pages;
  20         void *page_mem;
  21         const void *p;
  22
  23         if (!size)
  24                 return NULL;
  25
  26         /* Alloc file structure */
  27         lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL);
  28         if (!lm_file)
  29                 return NULL;
  30
  31         /* Create file */
  32         lm_file->filep =
  33                 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags);
  34         if (IS_ERR(lm_file->filep))
  35                 goto out_free_file;
  36
  37         stream_open(lm_file->filep->f_inode, lm_file->filep);
  38         mutex_init(&lm_file->lock);
  39
  40         /* prevent file from being released before we are done with it */
  41         get_file(lm_file->filep);
  42
  43         /* Allocate memory for file pages */
  44         npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE);
  45         pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);
  46         if (!pages)
  47                 goto out_put_file;
  48
  49         page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);
  50         if (!page_mem)
  51                 goto out_free_pages_array;
  52
  53         p = page_mem - offset_in_page(page_mem);
  54         for (unsigned long long i = 0; i < npages; i++) {
  55                 if (is_vmalloc_addr(p))
  56                         pages[i] = vmalloc_to_page(p);
  57                 else
  58                         pages[i] = kmap_to_page((void *)p);
  59                 if (!pages[i])
  60                         goto out_free_page_mem;
  61
  62                 p += PAGE_SIZE;
  63         }
  64
  65         /* Create scatterlist of file pages to use for DMA mapping later */
  66         if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0,
  67                                       size, GFP_KERNEL))
  68                 goto out_free_page_mem;
  69
  70         lm_file->size = size;
  71         lm_file->pages = pages;
  72         lm_file->npages = npages;
  73         lm_file->page_mem = page_mem;
  74         lm_file->alloc_size = npages * PAGE_SIZE;
  75
  76         return lm_file;
  77
  78 out_free_page_mem:
  79         kvfree(page_mem);
  80 out_free_pages_array:
  81         kfree(pages);
  82 out_put_file:
  83         fput(lm_file->filep);
  84         mutex_destroy(&lm_file->lock);
  85 out_free_file:
  86         kfree(lm_file);
  87
  88         return NULL;
  89 }
  90
  91 static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file)
  92 {
  93         mutex_lock(&lm_file->lock);
  94
  95         lm_file->disabled = true;
  96         lm_file->size = 0;
  97         lm_file->alloc_size = 0;
  98         lm_file->filep->f_pos = 0;
  99
 100         /* Free scatter list of file pages */
 101         sg_free_table(&lm_file->sg_table);
 102
 103         kvfree(lm_file->page_mem);
 104         lm_file->page_mem = NULL;
 105         kfree(lm_file->pages);
 106         lm_file->pages = NULL;
 107
 108         mutex_unlock(&lm_file->lock);
 109
 110         /* allow file to be released since we are done with it */
 111         fput(lm_file->filep);
 112 }
 113
 114 void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio)
 115 {
 116         if (!pds_vfio->save_file)
 117                 return;
 118
 119         pds_vfio_put_lm_file(pds_vfio->save_file);
 120         pds_vfio->save_file = NULL;
 121 }
 122
 123 void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio)
 124 {
 125         if (!pds_vfio->restore_file)
 126                 return;
 127
 128         pds_vfio_put_lm_file(pds_vfio->restore_file);
 129         pds_vfio->restore_file = NULL;
 130 }
 131
 132 static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file,
 133                                            unsigned long offset)
 134 {
 135         unsigned long cur_offset = 0;
 136         struct scatterlist *sg;
 137         unsigned int i;
 138
 139         /* All accesses are sequential */
 140         if (offset < lm_file->last_offset || !lm_file->last_offset_sg) {
 141                 lm_file->last_offset = 0;
 142                 lm_file->last_offset_sg = lm_file->sg_table.sgl;
 143                 lm_file->sg_last_entry = 0;
 144         }
 145
 146         cur_offset = lm_file->last_offset;
 147
 148         for_each_sg(lm_file->last_offset_sg, sg,
 149                     lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) {
 150                 if (offset < sg->length + cur_offset) {
 151                         lm_file->last_offset_sg = sg;
 152                         lm_file->sg_last_entry += i;
 153                         lm_file->last_offset = cur_offset;
 154                         return nth_page(sg_page(sg),
 155                                         (offset - cur_offset) / PAGE_SIZE);
 156                 }
 157                 cur_offset += sg->length;
 158         }
 159
 160         return NULL;
 161 }
 162
 163 static int pds_vfio_release_file(struct inode *inode, struct file *filp)
 164 {
 165         struct pds_vfio_lm_file *lm_file = filp->private_data;
 166
 167         mutex_lock(&lm_file->lock);
 168         lm_file->filep->f_pos = 0;
 169         lm_file->size = 0;
 170         mutex_unlock(&lm_file->lock);
 171         mutex_destroy(&lm_file->lock);
 172         kfree(lm_file);
 173
 174         return 0;
 175 }
 176
 177 static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf,
 178                                   size_t len, loff_t *pos)
 179 {
 180         struct pds_vfio_lm_file *lm_file = filp->private_data;
 181         ssize_t done = 0;
 182
 183         if (pos)
 184                 return -ESPIPE;
 185         pos = &filp->f_pos;
 186
 187         mutex_lock(&lm_file->lock);
 188
 189         if (lm_file->disabled) {
 190                 done = -ENODEV;
 191                 goto out_unlock;
 192         }
 193
 194         if (*pos > lm_file->size) {
 195                 done = -EINVAL;
 196                 goto out_unlock;
 197         }
 198
 199         len = min_t(size_t, lm_file->size - *pos, len);
 200         while (len) {
 201                 size_t page_offset;
 202                 struct page *page;
 203                 size_t page_len;
 204                 u8 *from_buff;
 205                 int err;
 206
 207                 page_offset = (*pos) % PAGE_SIZE;
 208                 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
 209                 if (!page) {
 210                         if (done == 0)
 211                                 done = -EINVAL;
 212                         goto out_unlock;
 213                 }
 214
 215                 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
 216                 from_buff = kmap_local_page(page);
 217                 err = copy_to_user(buf, from_buff + page_offset, page_len);
 218                 kunmap_local(from_buff);
 219                 if (err) {
 220                         done = -EFAULT;
 221                         goto out_unlock;
 222                 }
 223                 *pos += page_len;
 224                 len -= page_len;
 225                 done += page_len;
 226                 buf += page_len;
 227         }
 228
 229 out_unlock:
 230         mutex_unlock(&lm_file->lock);
 231         return done;
 232 }
 233
 234 static const struct file_operations pds_vfio_save_fops = {
 235         .owner = THIS_MODULE,
 236         .read = pds_vfio_save_read,
 237         .release = pds_vfio_release_file,
 238 };
 239
 240 static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio)
 241 {
 242         struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
 243         struct pds_vfio_lm_file *lm_file;
 244         u64 size;
 245         int err;
 246
 247         /* Get live migration state size in this state */
 248         err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size);
 249         if (err) {
 250                 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err));
 251                 return err;
 252         }
 253
 254         dev_dbg(dev, "save status, size = %lld\n", size);
 255
 256         if (!size) {
 257                 dev_err(dev, "invalid state size\n");
 258                 return -EIO;
 259         }
 260
 261         lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size);
 262         if (!lm_file) {
 263                 dev_err(dev, "failed to create save file\n");
 264                 return -ENOENT;
 265         }
 266
 267         dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n",
 268                 lm_file->size, lm_file->alloc_size, lm_file->npages);
 269
 270         pds_vfio->save_file = lm_file;
 271
 272         return 0;
 273 }
 274
 275 static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf,
 276                                       size_t len, loff_t *pos)
 277 {
 278         struct pds_vfio_lm_file *lm_file = filp->private_data;
 279         loff_t requested_length;
 280         ssize_t done = 0;
 281
 282         if (pos)
 283                 return -ESPIPE;
 284
 285         pos = &filp->f_pos;
 286
 287         if (*pos < 0 ||
 288             check_add_overflow((loff_t)len, *pos, &requested_length))
 289                 return -EINVAL;
 290
 291         mutex_lock(&lm_file->lock);
 292
 293         if (lm_file->disabled) {
 294                 done = -ENODEV;
 295                 goto out_unlock;
 296         }
 297
 298         while (len) {
 299                 size_t page_offset;
 300                 struct page *page;
 301                 size_t page_len;
 302                 u8 *to_buff;
 303                 int err;
 304
 305                 page_offset = (*pos) % PAGE_SIZE;
 306                 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);
 307                 if (!page) {
 308                         if (done == 0)
 309                                 done = -EINVAL;
 310                         goto out_unlock;
 311                 }
 312
 313                 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);
 314                 to_buff = kmap_local_page(page);
 315                 err = copy_from_user(to_buff + page_offset, buf, page_len);
 316                 kunmap_local(to_buff);
 317                 if (err) {
 318                         done = -EFAULT;
 319                         goto out_unlock;
 320                 }
 321                 *pos += page_len;
 322                 len -= page_len;
 323                 done += page_len;
 324                 buf += page_len;
 325                 lm_file->size += page_len;
 326         }
 327 out_unlock:
 328         mutex_unlock(&lm_file->lock);
 329         return done;
 330 }
 331
 332 static const struct file_operations pds_vfio_restore_fops = {
 333         .owner = THIS_MODULE,
 334         .write = pds_vfio_restore_write,
 335         .release = pds_vfio_release_file,
 336 };
 337
 338 static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio)
 339 {
 340         struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;
 341         struct pds_vfio_lm_file *lm_file;
 342         u64 size;
 343
 344         size = sizeof(union pds_lm_dev_state);
 345         dev_dbg(dev, "restore status, size = %lld\n", size);
 346
 347         if (!size) {
 348                 dev_err(dev, "invalid state size");
 349                 return -EIO;
 350         }
 351
 352         lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size);
 353         if (!lm_file) {
 354                 dev_err(dev, "failed to create restore file");
 355                 return -ENOENT;
 356         }
 357         pds_vfio->restore_file = lm_file;
 358
 359         return 0;
 360 }
 361
 362 struct file *
 363 pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio,
 364                                   enum vfio_device_mig_state next)
 365 {
 366         enum vfio_device_mig_state cur = pds_vfio->state;
 367         int err;
 368
 369         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) {
 370                 err = pds_vfio_get_save_file(pds_vfio);
 371                 if (err)
 372                         return ERR_PTR(err);
 373
 374                 err = pds_vfio_get_lm_state_cmd(pds_vfio);
 375                 if (err) {
 376                         pds_vfio_put_save_file(pds_vfio);
 377                         return ERR_PTR(err);
 378                 }
 379
 380                 return pds_vfio->save_file->filep;
 381         }
 382
 383         if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) {
 384                 pds_vfio_put_save_file(pds_vfio);
 385                 pds_vfio_dirty_disable(pds_vfio, true);
 386                 return NULL;
 387         }
 388
 389         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) {
 390                 err = pds_vfio_get_restore_file(pds_vfio);
 391                 if (err)
 392                         return ERR_PTR(err);
 393
 394                 return pds_vfio->restore_file->filep;
 395         }
 396
 397         if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) {
 398                 err = pds_vfio_set_lm_state_cmd(pds_vfio);
 399                 if (err)
 400                         return ERR_PTR(err);
 401
 402                 pds_vfio_put_restore_file(pds_vfio);
 403                 return NULL;
 404         }
 405
 406         if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
 407                 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio,
 408                                                     PDS_LM_STA_IN_PROGRESS);
 409                 err = pds_vfio_suspend_device_cmd(pds_vfio,
 410                                                   PDS_LM_SUSPEND_RESUME_TYPE_P2P);
 411                 if (err)
 412                         return ERR_PTR(err);
 413
 414                 return NULL;
 415         }
 416
 417         if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) {
 418                 err = pds_vfio_resume_device_cmd(pds_vfio,
 419                                                  PDS_LM_SUSPEND_RESUME_TYPE_FULL);
 420                 if (err)
 421                         return ERR_PTR(err);
 422
 423                 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);
 424                 return NULL;
 425         }
 426
 427         if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) {
 428                 err = pds_vfio_resume_device_cmd(pds_vfio,
 429                                                  PDS_LM_SUSPEND_RESUME_TYPE_P2P);
 430                 if (err)
 431                         return ERR_PTR(err);
 432
 433                 return NULL;
 434         }
 435
 436         if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) {
 437                 err = pds_vfio_suspend_device_cmd(pds_vfio,
 438                                                   PDS_LM_SUSPEND_RESUME_TYPE_FULL);
 439                 if (err)
 440                         return ERR_PTR(err);
 441                 return NULL;
 442         }
 443
 444         return ERR_PTR(-EINVAL);
 445 }