hw/msix.c

   1 /*
   2  * MSI-X device support
   3  *
   4  * This module includes support for MSI-X in pci devices.
   5  *
   6  * Author: Michael S. Tsirkin <mst@redhat.com>
   7  *
   8  *  Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2.  See
  11  * the COPYING file in the top-level directory.
  12  */
  13
  14 #include "hw.h"
  15 #include "msix.h"
  16 #include "pci.h"
  17 #include "range.h"
  18 #include "kvm.h"
  19
  20 /* MSI-X capability structure */
  21 #define MSIX_TABLE_OFFSET 4
  22 #define MSIX_PBA_OFFSET 8
  23 #define MSIX_CAP_LENGTH 12
  24
  25 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
  26 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
  27 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
  28 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
  29
  30 /* MSI-X table format */
  31 #define MSIX_MSG_ADDR 0
  32 #define MSIX_MSG_UPPER_ADDR 4
  33 #define MSIX_MSG_DATA 8
  34 #define MSIX_VECTOR_CTRL 12
  35 #define MSIX_ENTRY_SIZE 16
  36 #define MSIX_VECTOR_MASK 0x1
  37
  38 /* How much space does an MSIX table need. */
  39 /* The spec requires giving the table structure
  40  * a 4K aligned region all by itself. */
  41 #define MSIX_PAGE_SIZE 0x1000
  42 /* Reserve second half of the page for pending bits */
  43 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
  44 #define MSIX_MAX_ENTRIES 32
  45
  46
  47 /* Flag for interrupt controller to declare MSI-X support */
  48 int msix_supported;
  49
  50 /* KVM specific MSIX helpers */
  51 static void kvm_msix_free(PCIDevice *dev)
  52 {
  53     int vector, changed = 0;
  54
  55     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
  56         if (dev->msix_entry_used[vector]) {
  57             kvm_msi_message_del(&dev->msix_irq_entries[vector]);
  58             changed = 1;
  59         }
  60     }
  61     if (changed) {
  62         kvm_commit_irq_routes();
  63     }
  64 }
  65
  66 static void kvm_msix_message_from_vector(PCIDevice *dev, unsigned vector,
  67                                          KVMMsiMessage *kmm)
  68 {
  69     uint8_t *table_entry = dev->msix_table_page + vector * MSIX_ENTRY_SIZE;
  70
  71     kmm->addr_lo = pci_get_long(table_entry + MSIX_MSG_ADDR);
  72     kmm->addr_hi = pci_get_long(table_entry + MSIX_MSG_UPPER_ADDR);
  73     kmm->data = pci_get_long(table_entry + MSIX_MSG_DATA);
  74 }
  75
  76 static void kvm_msix_update(PCIDevice *dev, int vector,
  77                             int was_masked, int is_masked)
  78 {
  79     KVMMsiMessage new_entry, *entry;
  80     int mask_cleared = was_masked && !is_masked;
  81     int r;
  82
  83     /* It is only legal to change an entry when it is masked. Therefore, it is
  84      * enough to update the routing in kernel when mask is being cleared. */
  85     if (!mask_cleared) {
  86         return;
  87     }
  88     if (!dev->msix_entry_used[vector]) {
  89         return;
  90     }
  91
  92     entry = dev->msix_irq_entries + vector;
  93     kvm_msix_message_from_vector(dev, vector, &new_entry);
  94     r = kvm_msi_message_update(entry, &new_entry);
  95     if (r < 0) {
  96         fprintf(stderr, "%s: kvm_update_msix failed: %s\n", __func__,
  97                 strerror(-r));
  98         exit(1);
  99     }
 100     if (r > 0) {
 101         *entry = new_entry;
 102         r = kvm_commit_irq_routes();
 103         if (r) {
 104             fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__,
 105                     strerror(-r));
 106             exit(1);
 107         }
 108     }
 109 }
 110
 111 static int kvm_msix_vector_add(PCIDevice *dev, unsigned vector)
 112 {
 113     KVMMsiMessage *kmm = dev->msix_irq_entries + vector;
 114     int r;
 115
 116     kvm_msix_message_from_vector(dev, vector, kmm);
 117     r = kvm_msi_message_add(kmm);
 118     if (r < 0) {
 119         fprintf(stderr, "%s: kvm_add_msix failed: %s\n", __func__, strerror(-r));
 120         return r;
 121     }
 122
 123     r = kvm_commit_irq_routes();
 124     if (r < 0) {
 125         fprintf(stderr, "%s: kvm_commit_irq_routes failed: %s\n", __func__, strerror(-r));
 126         return r;
 127     }
 128     return 0;
 129 }
 130
 131 static void kvm_msix_vector_del(PCIDevice *dev, unsigned vector)
 132 {
 133     kvm_msi_message_del(&dev->msix_irq_entries[vector]);
 134     kvm_commit_irq_routes();
 135 }
 136
 137 /* Add MSI-X capability to the config space for the device. */
 138 /* Given a bar and its size, add MSI-X table on top of it
 139  * and fill MSI-X capability in the config space.
 140  * Original bar size must be a power of 2 or 0.
 141  * New bar size is returned. */
 142 static int msix_add_config(struct PCIDevice *pdev, unsigned short nentries,
 143                            unsigned bar_nr, unsigned bar_size)
 144 {
 145     int config_offset;
 146     uint8_t *config;
 147
 148     pdev->msix_bar_size = bar_size;
 149
 150     config_offset = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
 151
 152     if (!config_offset) {
 153         uint32_t new_size;
 154
 155         if (nentries < 1 || nentries > PCI_MSIX_FLAGS_QSIZE + 1)
 156             return -EINVAL;
 157         if (bar_size > 0x80000000)
 158             return -ENOSPC;
 159
 160         /* Add space for MSI-X structures */
 161         if (!bar_size) {
 162             new_size = MSIX_PAGE_SIZE;
 163         } else if (bar_size < MSIX_PAGE_SIZE) {
 164             bar_size = MSIX_PAGE_SIZE;
 165             new_size = MSIX_PAGE_SIZE * 2;
 166         } else {
 167             new_size = bar_size * 2;
 168         }
 169
 170         pdev->msix_bar_size = new_size;
 171         config_offset = pci_add_capability(pdev, PCI_CAP_ID_MSIX,
 172                                            0, MSIX_CAP_LENGTH);
 173         if (config_offset < 0)
 174             return config_offset;
 175         config = pdev->config + config_offset;
 176
 177         pci_set_word(config + PCI_MSIX_FLAGS, nentries - 1);
 178         /* Table on top of BAR */
 179         pci_set_long(config + MSIX_TABLE_OFFSET, bar_size | bar_nr);
 180         /* Pending bits on top of that */
 181         pci_set_long(config + MSIX_PBA_OFFSET, (bar_size + MSIX_PAGE_PENDING) |
 182                      bar_nr);
 183     }
 184     pdev->msix_cap = config_offset;
 185     /* Make flags bit writable. */
 186     pdev->wmask[config_offset + MSIX_CONTROL_OFFSET] |= MSIX_ENABLE_MASK |
 187             MSIX_MASKALL_MASK;
 188     return 0;
 189 }
 190
 191 static uint32_t msix_mmio_readl(void *opaque, target_phys_addr_t addr)
 192 {
 193     PCIDevice *dev = opaque;
 194     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
 195     void *page = dev->msix_table_page;
 196
 197     return pci_get_long(page + offset);
 198 }
 199
 200 static uint32_t msix_mmio_read_unallowed(void *opaque, target_phys_addr_t addr)
 201 {
 202     fprintf(stderr, "MSI-X: only dword read is allowed!\n");
 203     return 0;
 204 }
 205
 206 static uint8_t msix_pending_mask(int vector)
 207 {
 208     return 1 << (vector % 8);
 209 }
 210
 211 static uint8_t *msix_pending_byte(PCIDevice *dev, int vector)
 212 {
 213     return dev->msix_table_page + MSIX_PAGE_PENDING + vector / 8;
 214 }
 215
 216 static int msix_is_pending(PCIDevice *dev, int vector)
 217 {
 218     return *msix_pending_byte(dev, vector) & msix_pending_mask(vector);
 219 }
 220
 221 static void msix_set_pending(PCIDevice *dev, int vector)
 222 {
 223     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 224 }
 225
 226 static void msix_clr_pending(PCIDevice *dev, int vector)
 227 {
 228     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 229 }
 230
 231 static int msix_function_masked(PCIDevice *dev)
 232 {
 233     return dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] & MSIX_MASKALL_MASK;
 234 }
 235
 236 static int msix_is_masked(PCIDevice *dev, int vector)
 237 {
 238     unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 239     return msix_function_masked(dev) ||
 240            dev->msix_table_page[offset] & MSIX_VECTOR_MASK;
 241 }
 242
 243 static void msix_handle_mask_update(PCIDevice *dev, int vector)
 244 {
 245     if (!msix_is_masked(dev, vector) && msix_is_pending(dev, vector)) {
 246         msix_clr_pending(dev, vector);
 247         msix_notify(dev, vector);
 248     }
 249 }
 250
 251 /* Handle MSI-X capability config write. */
 252 void msix_write_config(PCIDevice *dev, uint32_t addr,
 253                        uint32_t val, int len)
 254 {
 255     unsigned enable_pos = dev->msix_cap + MSIX_CONTROL_OFFSET;
 256     int vector;
 257
 258     if (!range_covers_byte(addr, len, enable_pos)) {
 259         return;
 260     }
 261
 262     if (!msix_enabled(dev)) {
 263         return;
 264     }
 265
 266     pci_device_deassert_intx(dev);
 267
 268     if (msix_function_masked(dev)) {
 269         return;
 270     }
 271
 272     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 273         msix_handle_mask_update(dev, vector);
 274     }
 275 }
 276
 277 static void msix_mmio_writel(void *opaque, target_phys_addr_t addr,
 278                              uint32_t val)
 279 {
 280     PCIDevice *dev = opaque;
 281     unsigned int offset = addr & (MSIX_PAGE_SIZE - 1) & ~0x3;
 282     int vector = offset / MSIX_ENTRY_SIZE;
 283     int was_masked = msix_is_masked(dev, vector);
 284     pci_set_long(dev->msix_table_page + offset, val);
 285     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 286         kvm_msix_update(dev, vector, was_masked, msix_is_masked(dev, vector));
 287     }
 288     if (was_masked != msix_is_masked(dev, vector) && dev->msix_mask_notifier) {
 289         int r = dev->msix_mask_notifier(dev, vector,
 290                                         msix_is_masked(dev, vector));
 291         assert(r >= 0);
 292     }
 293     msix_handle_mask_update(dev, vector);
 294 }
 295
 296 static void msix_mmio_write_unallowed(void *opaque, target_phys_addr_t addr,
 297                                       uint32_t val)
 298 {
 299     fprintf(stderr, "MSI-X: only dword write is allowed!\n");
 300 }
 301
 302 static CPUWriteMemoryFunc * const msix_mmio_write[] = {
 303     msix_mmio_write_unallowed, msix_mmio_write_unallowed, msix_mmio_writel
 304 };
 305
 306 static CPUReadMemoryFunc * const msix_mmio_read[] = {
 307     msix_mmio_read_unallowed, msix_mmio_read_unallowed, msix_mmio_readl
 308 };
 309
 310 /* Should be called from device's map method. */
 311 void msix_mmio_map(PCIDevice *d, int region_num,
 312                    pcibus_t addr, pcibus_t size, int type)
 313 {
 314     uint8_t *config = d->config + d->msix_cap;
 315     uint32_t table = pci_get_long(config + MSIX_TABLE_OFFSET);
 316     uint32_t offset = table & ~(MSIX_PAGE_SIZE - 1);
 317     /* TODO: for assigned devices, we'll want to make it possible to map
 318      * pending bits separately in case they are in a separate bar. */
 319     int table_bir = table & PCI_MSIX_FLAGS_BIRMASK;
 320
 321     if (table_bir != region_num)
 322         return;
 323     if (size <= offset)
 324         return;
 325     cpu_register_physical_memory(addr + offset,
 326                                  MIN(size - offset, MSIX_PAGE_SIZE),
 327                                  d->msix_mmio_index);
 328 }
 329
 330 static void msix_mask_all(struct PCIDevice *dev, unsigned nentries)
 331 {
 332     int vector, r;
 333     for (vector = 0; vector < nentries; ++vector) {
 334         unsigned offset = vector * MSIX_ENTRY_SIZE + MSIX_VECTOR_CTRL;
 335         int was_masked = msix_is_masked(dev, vector);
 336         dev->msix_table_page[offset] |= MSIX_VECTOR_MASK;
 337         if (was_masked != msix_is_masked(dev, vector) &&
 338             dev->msix_mask_notifier) {
 339             r = dev->msix_mask_notifier(dev, vector,
 340                                         msix_is_masked(dev, vector));
 341             assert(r >= 0);
 342         }
 343     }
 344 }
 345
 346 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
 347  * modified, it should be retrieved with msix_bar_size. */
 348 int msix_init(struct PCIDevice *dev, unsigned short nentries,
 349               unsigned bar_nr, unsigned bar_size)
 350 {
 351     int ret;
 352     /* Nothing to do if MSI is not supported by interrupt controller */
 353     if (!msix_supported ||
 354         (kvm_enabled() && kvm_irqchip_in_kernel() && !kvm_has_gsi_routing())) {
 355         return -ENOTSUP;
 356     }
 357
 358     if (nentries > MSIX_MAX_ENTRIES)
 359         return -EINVAL;
 360
 361     dev->msix_mask_notifier = NULL;
 362     dev->msix_entry_used = qemu_mallocz(MSIX_MAX_ENTRIES *
 363                                         sizeof *dev->msix_entry_used);
 364
 365     dev->msix_table_page = qemu_mallocz(MSIX_PAGE_SIZE);
 366     msix_mask_all(dev, nentries);
 367
 368     dev->msix_mmio_index = cpu_register_io_memory(msix_mmio_read,
 369                                                   msix_mmio_write, dev,
 370                                                   DEVICE_NATIVE_ENDIAN);
 371     if (dev->msix_mmio_index == -1) {
 372         ret = -EBUSY;
 373         goto err_index;
 374     }
 375
 376     dev->msix_entries_nr = nentries;
 377     ret = msix_add_config(dev, nentries, bar_nr, bar_size);
 378     if (ret)
 379         goto err_config;
 380
 381     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 382         dev->msix_irq_entries = qemu_malloc(nentries *
 383                                             sizeof *dev->msix_irq_entries);
 384     }
 385
 386     dev->cap_present |= QEMU_PCI_CAP_MSIX;
 387     return 0;
 388
 389 err_config:
 390     dev->msix_entries_nr = 0;
 391     cpu_unregister_io_memory(dev->msix_mmio_index);
 392 err_index:
 393     qemu_free(dev->msix_table_page);
 394     dev->msix_table_page = NULL;
 395     qemu_free(dev->msix_entry_used);
 396     dev->msix_entry_used = NULL;
 397     return ret;
 398 }
 399
 400 static void msix_free_irq_entries(PCIDevice *dev)
 401 {
 402     int vector;
 403
 404     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 405         kvm_msix_free(dev);
 406     }
 407
 408     for (vector = 0; vector < dev->msix_entries_nr; ++vector) {
 409         dev->msix_entry_used[vector] = 0;
 410         msix_clr_pending(dev, vector);
 411     }
 412 }
 413
 414 /* Clean up resources for the device. */
 415 int msix_uninit(PCIDevice *dev)
 416 {
 417     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 418         return 0;
 419     pci_del_capability(dev, PCI_CAP_ID_MSIX, MSIX_CAP_LENGTH);
 420     dev->msix_cap = 0;
 421     msix_free_irq_entries(dev);
 422     dev->msix_entries_nr = 0;
 423     cpu_unregister_io_memory(dev->msix_mmio_index);
 424     qemu_free(dev->msix_table_page);
 425     dev->msix_table_page = NULL;
 426     qemu_free(dev->msix_entry_used);
 427     dev->msix_entry_used = NULL;
 428     qemu_free(dev->msix_irq_entries);
 429     dev->msix_irq_entries = NULL;
 430     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
 431     return 0;
 432 }
 433
 434 void msix_save(PCIDevice *dev, QEMUFile *f)
 435 {
 436     unsigned n = dev->msix_entries_nr;
 437
 438     if (!msix_supported) {
 439         return;
 440     }
 441
 442     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 443         return;
 444     }
 445     qemu_put_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 446     qemu_put_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 447 }
 448
 449 /* Should be called after restoring the config space. */
 450 void msix_load(PCIDevice *dev, QEMUFile *f)
 451 {
 452     unsigned n = dev->msix_entries_nr;
 453
 454     if (!msix_supported)
 455         return;
 456
 457     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX)) {
 458         return;
 459     }
 460
 461     msix_free_irq_entries(dev);
 462     qemu_get_buffer(f, dev->msix_table_page, n * MSIX_ENTRY_SIZE);
 463     qemu_get_buffer(f, dev->msix_table_page + MSIX_PAGE_PENDING, (n + 7) / 8);
 464 }
 465
 466 /* Does device support MSI-X? */
 467 int msix_present(PCIDevice *dev)
 468 {
 469     return dev->cap_present & QEMU_PCI_CAP_MSIX;
 470 }
 471
 472 /* Is MSI-X enabled? */
 473 int msix_enabled(PCIDevice *dev)
 474 {
 475     return (dev->cap_present & QEMU_PCI_CAP_MSIX) &&
 476         (dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &
 477          MSIX_ENABLE_MASK);
 478 }
 479
 480 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
 481 uint32_t msix_bar_size(PCIDevice *dev)
 482 {
 483     return (dev->cap_present & QEMU_PCI_CAP_MSIX) ?
 484         dev->msix_bar_size : 0;
 485 }
 486
 487 /* Send an MSI-X message */
 488 void msix_notify(PCIDevice *dev, unsigned vector)
 489 {
 490     uint8_t *table_entry = dev->msix_table_page + vector * MSIX_ENTRY_SIZE;
 491     uint64_t address;
 492     uint32_t data;
 493
 494     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector])
 495         return;
 496     if (msix_is_masked(dev, vector)) {
 497         msix_set_pending(dev, vector);
 498         return;
 499     }
 500
 501     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 502         kvm_set_irq(dev->msix_irq_entries[vector].gsi, 1, NULL);
 503         return;
 504     }
 505
 506     address = pci_get_long(table_entry + MSIX_MSG_UPPER_ADDR);
 507     address = (address << 32) | pci_get_long(table_entry + MSIX_MSG_ADDR);
 508     data = pci_get_long(table_entry + MSIX_MSG_DATA);
 509     stl_phys(address, data);
 510 }
 511
 512 void msix_reset(PCIDevice *dev)
 513 {
 514     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 515         return;
 516     msix_free_irq_entries(dev);
 517     dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
 518             ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
 519     memset(dev->msix_table_page, 0, MSIX_PAGE_SIZE);
 520     msix_mask_all(dev, dev->msix_entries_nr);
 521 }
 522
 523 /* PCI spec suggests that devices make it possible for software to configure
 524  * less vectors than supported by the device, but does not specify a standard
 525  * mechanism for devices to do so.
 526  *
 527  * We support this by asking devices to declare vectors software is going to
 528  * actually use, and checking this on the notification path. Devices that
 529  * don't want to follow the spec suggestion can declare all vectors as used. */
 530
 531 /* Mark vector as used. */
 532 int msix_vector_use(PCIDevice *dev, unsigned vector)
 533 {
 534     int ret;
 535     if (vector >= dev->msix_entries_nr)
 536         return -EINVAL;
 537     if (kvm_enabled() && kvm_irqchip_in_kernel() &&
 538         !dev->msix_entry_used[vector]) {
 539         ret = kvm_msix_vector_add(dev, vector);
 540         if (ret) {
 541             return ret;
 542         }
 543     }
 544     ++dev->msix_entry_used[vector];
 545     return 0;
 546 }
 547
 548 /* Mark vector as unused. */
 549 void msix_vector_unuse(PCIDevice *dev, unsigned vector)
 550 {
 551     if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) {
 552         return;
 553     }
 554     if (--dev->msix_entry_used[vector]) {
 555         return;
 556     }
 557     if (kvm_enabled() && kvm_irqchip_in_kernel()) {
 558         kvm_msix_vector_del(dev, vector);
 559     }
 560     msix_clr_pending(dev, vector);
 561 }
 562
 563 void msix_unuse_all_vectors(PCIDevice *dev)
 564 {
 565     if (!(dev->cap_present & QEMU_PCI_CAP_MSIX))
 566         return;
 567     msix_free_irq_entries(dev);
 568 }
 569
 570 /* Invoke the notifier if vector entry is used and unmasked. */
 571 static int msix_notify_if_unmasked(PCIDevice *dev, unsigned vector, int masked)
 572 {
 573     assert(dev->msix_mask_notifier);
 574     if (!dev->msix_entry_used[vector] || msix_is_masked(dev, vector)) {
 575         return 0;
 576     }
 577     return dev->msix_mask_notifier(dev, vector, masked);
 578 }
 579
 580 static int msix_set_mask_notifier_for_vector(PCIDevice *dev, unsigned vector)
 581 {
 582         /* Notifier has been set. Invoke it on unmasked vectors. */
 583         return msix_notify_if_unmasked(dev, vector, 0);
 584 }
 585
 586 static int msix_unset_mask_notifier_for_vector(PCIDevice *dev, unsigned vector)
 587 {
 588         /* Notifier will be unset. Invoke it to mask unmasked entries. */
 589         return msix_notify_if_unmasked(dev, vector, 1);
 590 }
 591
 592 int msix_set_mask_notifier(PCIDevice *dev, msix_mask_notifier_func f)
 593 {
 594     int r, n;
 595     assert(!dev->msix_mask_notifier);
 596     dev->msix_mask_notifier = f;
 597     for (n = 0; n < dev->msix_entries_nr; ++n) {
 598         r = msix_set_mask_notifier_for_vector(dev, n);
 599         if (r < 0) {
 600             goto undo;
 601         }
 602     }
 603     return 0;
 604
 605 undo:
 606     while (--n >= 0) {
 607         msix_unset_mask_notifier_for_vector(dev, n);
 608     }
 609     dev->msix_mask_notifier = NULL;
 610     return r;
 611 }
 612
 613 int msix_unset_mask_notifier(PCIDevice *dev)
 614 {
 615     int r, n;
 616     assert(dev->msix_mask_notifier);
 617     for (n = 0; n < dev->msix_entries_nr; ++n) {
 618         r = msix_unset_mask_notifier_for_vector(dev, n);
 619         if (r < 0) {
 620             goto undo;
 621         }
 622     }
 623     dev->msix_mask_notifier = NULL;
 624     return 0;
 625
 626 undo:
 627     while (--n >= 0) {
 628         msix_set_mask_notifier_for_vector(dev, n);
 629     }
 630     return r;
 631 }