drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36
  37 #undef PREFIX
  38 #define PREFIX "DMAR:"
  39
  40 /* No locks are needed as DMA remapping hardware unit
  41  * list is constructed at boot time and hotplug of
  42  * these units are not supported by the architecture.
  43  */
  44 LIST_HEAD(dmar_drhd_units);
  45
  46 static struct acpi_table_header * __initdata dmar_tbl;
  47 static acpi_size dmar_tbl_size;
  48
  49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  50 {
  51         /*
  52          * add INCLUDE_ALL at the tail, so scan the list will find it at
  53          * the very end.
  54          */
  55         if (drhd->include_all)
  56                 list_add_tail(&drhd->list, &dmar_drhd_units);
  57         else
  58                 list_add(&drhd->list, &dmar_drhd_units);
  59 }
  60
  61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  62                                            struct pci_dev **dev, u16 segment)
  63 {
  64         struct pci_bus *bus;
  65         struct pci_dev *pdev = NULL;
  66         struct acpi_dmar_pci_path *path;
  67         int count;
  68
  69         bus = pci_find_bus(segment, scope->bus);
  70         path = (struct acpi_dmar_pci_path *)(scope + 1);
  71         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  72                 / sizeof(struct acpi_dmar_pci_path);
  73
  74         while (count) {
  75                 if (pdev)
  76                         pci_dev_put(pdev);
  77                 /*
  78                  * Some BIOSes list non-exist devices in DMAR table, just
  79                  * ignore it
  80                  */
  81                 if (!bus) {
  82                         printk(KERN_WARNING
  83                         PREFIX "Device scope bus [%d] not found\n",
  84                         scope->bus);
  85                         break;
  86                 }
  87                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  88                 if (!pdev) {
  89                         printk(KERN_WARNING PREFIX
  90                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  91                                 segment, bus->number, path->dev, path->fn);
  92                         break;
  93                 }
  94                 path ++;
  95                 count --;
  96                 bus = pdev->subordinate;
  97         }
  98         if (!pdev) {
  99                 printk(KERN_WARNING PREFIX
 100                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 101                 segment, scope->bus, path->dev, path->fn);
 102                 *dev = NULL;
 103                 return 0;
 104         }
 105         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 106                         pdev->subordinate) || (scope->entry_type == \
 107                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 108                 pci_dev_put(pdev);
 109                 printk(KERN_WARNING PREFIX
 110                         "Device scope type does not match for %s\n",
 111                          pci_name(pdev));
 112                 return -EINVAL;
 113         }
 114         *dev = pdev;
 115         return 0;
 116 }
 117
 118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 119                                        struct pci_dev ***devices, u16 segment)
 120 {
 121         struct acpi_dmar_device_scope *scope;
 122         void * tmp = start;
 123         int index;
 124         int ret;
 125
 126         *cnt = 0;
 127         while (start < end) {
 128                 scope = start;
 129                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 130                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 131                         (*cnt)++;
 132                 else
 133                         printk(KERN_WARNING PREFIX
 134                                 "Unsupported device scope\n");
 135                 start += scope->length;
 136         }
 137         if (*cnt == 0)
 138                 return 0;
 139
 140         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 141         if (!*devices)
 142                 return -ENOMEM;
 143
 144         start = tmp;
 145         index = 0;
 146         while (start < end) {
 147                 scope = start;
 148                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 149                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 150                         ret = dmar_parse_one_dev_scope(scope,
 151                                 &(*devices)[index], segment);
 152                         if (ret) {
 153                                 kfree(*devices);
 154                                 return ret;
 155                         }
 156                         index ++;
 157                 }
 158                 start += scope->length;
 159         }
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 166  * structure which uniquely represent one DMA remapping hardware unit
 167  * present in the platform
 168  */
 169 static int __init
 170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 171 {
 172         struct acpi_dmar_hardware_unit *drhd;
 173         struct dmar_drhd_unit *dmaru;
 174         int ret = 0;
 175
 176         drhd = (struct acpi_dmar_hardware_unit *)header;
 177         if (!drhd->address) {
 178                 /* Promote an attitude of violence to a BIOS engineer today */
 179                 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
 180                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 181                      dmi_get_system_info(DMI_BIOS_VENDOR),
 182                      dmi_get_system_info(DMI_BIOS_VERSION),
 183                      dmi_get_system_info(DMI_PRODUCT_VERSION));
 184                 return -ENODEV;
 185         }
 186         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 187         if (!dmaru)
 188                 return -ENOMEM;
 189
 190         dmaru->hdr = header;
 191         dmaru->reg_base_addr = drhd->address;
 192         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 193
 194         ret = alloc_iommu(dmaru);
 195         if (ret) {
 196                 kfree(dmaru);
 197                 return ret;
 198         }
 199         dmar_register_drhd_unit(dmaru);
 200         return 0;
 201 }
 202
 203 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 204 {
 205         struct acpi_dmar_hardware_unit *drhd;
 206         int ret = 0;
 207
 208         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 209
 210         if (dmaru->include_all)
 211                 return 0;
 212
 213         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 214                                 ((void *)drhd) + drhd->header.length,
 215                                 &dmaru->devices_cnt, &dmaru->devices,
 216                                 drhd->segment);
 217         if (ret) {
 218                 list_del(&dmaru->list);
 219                 kfree(dmaru);
 220         }
 221         return ret;
 222 }
 223
 224 #ifdef CONFIG_DMAR
 225 LIST_HEAD(dmar_rmrr_units);
 226
 227 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 228 {
 229         list_add(&rmrr->list, &dmar_rmrr_units);
 230 }
 231
 232
 233 static int __init
 234 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 235 {
 236         struct acpi_dmar_reserved_memory *rmrr;
 237         struct dmar_rmrr_unit *rmrru;
 238
 239         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 240         if (!rmrru)
 241                 return -ENOMEM;
 242
 243         rmrru->hdr = header;
 244         rmrr = (struct acpi_dmar_reserved_memory *)header;
 245         rmrru->base_address = rmrr->base_address;
 246         rmrru->end_address = rmrr->end_address;
 247
 248         dmar_register_rmrr_unit(rmrru);
 249         return 0;
 250 }
 251
 252 static int __init
 253 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 254 {
 255         struct acpi_dmar_reserved_memory *rmrr;
 256         int ret;
 257
 258         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 259         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 260                 ((void *)rmrr) + rmrr->header.length,
 261                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 262
 263         if (ret || (rmrru->devices_cnt == 0)) {
 264                 list_del(&rmrru->list);
 265                 kfree(rmrru);
 266         }
 267         return ret;
 268 }
 269 #endif
 270
 271 static void __init
 272 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 273 {
 274         struct acpi_dmar_hardware_unit *drhd;
 275         struct acpi_dmar_reserved_memory *rmrr;
 276
 277         switch (header->type) {
 278         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 279                 drhd = (struct acpi_dmar_hardware_unit *)header;
 280                 printk (KERN_INFO PREFIX
 281                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
 282                         drhd->flags, (unsigned long long)drhd->address);
 283                 break;
 284         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 285                 rmrr = (struct acpi_dmar_reserved_memory *)header;
 286
 287                 printk (KERN_INFO PREFIX
 288                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
 289                         (unsigned long long)rmrr->base_address,
 290                         (unsigned long long)rmrr->end_address);
 291                 break;
 292         }
 293 }
 294
 295 /**
 296  * dmar_table_detect - checks to see if the platform supports DMAR devices
 297  */
 298 static int __init dmar_table_detect(void)
 299 {
 300         acpi_status status = AE_OK;
 301
 302         /* if we could find DMAR table, then there are DMAR devices */
 303         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 304                                 (struct acpi_table_header **)&dmar_tbl,
 305                                 &dmar_tbl_size);
 306
 307         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 308                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 309                 status = AE_NOT_FOUND;
 310         }
 311
 312         return (ACPI_SUCCESS(status) ? 1 : 0);
 313 }
 314
 315 /**
 316  * parse_dmar_table - parses the DMA reporting table
 317  */
 318 static int __init
 319 parse_dmar_table(void)
 320 {
 321         struct acpi_table_dmar *dmar;
 322         struct acpi_dmar_header *entry_header;
 323         int ret = 0;
 324
 325         /*
 326          * Do it again, earlier dmar_tbl mapping could be mapped with
 327          * fixed map.
 328          */
 329         dmar_table_detect();
 330
 331         dmar = (struct acpi_table_dmar *)dmar_tbl;
 332         if (!dmar)
 333                 return -ENODEV;
 334
 335         if (dmar->width < PAGE_SHIFT - 1) {
 336                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 337                 return -EINVAL;
 338         }
 339
 340         printk (KERN_INFO PREFIX "Host address width %d\n",
 341                 dmar->width + 1);
 342
 343         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 344         while (((unsigned long)entry_header) <
 345                         (((unsigned long)dmar) + dmar_tbl->length)) {
 346                 /* Avoid looping forever on bad ACPI tables */
 347                 if (entry_header->length == 0) {
 348                         printk(KERN_WARNING PREFIX
 349                                 "Invalid 0-length structure\n");
 350                         ret = -EINVAL;
 351                         break;
 352                 }
 353
 354                 dmar_table_print_dmar_entry(entry_header);
 355
 356                 switch (entry_header->type) {
 357                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 358                         ret = dmar_parse_one_drhd(entry_header);
 359                         break;
 360                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 361 #ifdef CONFIG_DMAR
 362                         ret = dmar_parse_one_rmrr(entry_header);
 363 #endif
 364                         break;
 365                 default:
 366                         printk(KERN_WARNING PREFIX
 367                                 "Unknown DMAR structure type\n");
 368                         ret = 0; /* for forward compatibility */
 369                         break;
 370                 }
 371                 if (ret)
 372                         break;
 373
 374                 entry_header = ((void *)entry_header + entry_header->length);
 375         }
 376         return ret;
 377 }
 378
 379 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 380                           struct pci_dev *dev)
 381 {
 382         int index;
 383
 384         while (dev) {
 385                 for (index = 0; index < cnt; index++)
 386                         if (dev == devices[index])
 387                                 return 1;
 388
 389                 /* Check our parent */
 390                 dev = dev->bus->self;
 391         }
 392
 393         return 0;
 394 }
 395
 396 struct dmar_drhd_unit *
 397 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 398 {
 399         struct dmar_drhd_unit *dmaru = NULL;
 400         struct acpi_dmar_hardware_unit *drhd;
 401
 402         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 403                 drhd = container_of(dmaru->hdr,
 404                                     struct acpi_dmar_hardware_unit,
 405                                     header);
 406
 407                 if (dmaru->include_all &&
 408                     drhd->segment == pci_domain_nr(dev->bus))
 409                         return dmaru;
 410
 411                 if (dmar_pci_device_match(dmaru->devices,
 412                                           dmaru->devices_cnt, dev))
 413                         return dmaru;
 414         }
 415
 416         return NULL;
 417 }
 418
 419 int __init dmar_dev_scope_init(void)
 420 {
 421         struct dmar_drhd_unit *drhd, *drhd_n;
 422         int ret = -ENODEV;
 423
 424         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 425                 ret = dmar_parse_dev(drhd);
 426                 if (ret)
 427                         return ret;
 428         }
 429
 430 #ifdef CONFIG_DMAR
 431         {
 432                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 433                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 434                         ret = rmrr_parse_dev(rmrr);
 435                         if (ret)
 436                                 return ret;
 437                 }
 438         }
 439 #endif
 440
 441         return ret;
 442 }
 443
 444
 445 int __init dmar_table_init(void)
 446 {
 447         static int dmar_table_initialized;
 448         int ret;
 449
 450         if (dmar_table_initialized)
 451                 return 0;
 452
 453         dmar_table_initialized = 1;
 454
 455         ret = parse_dmar_table();
 456         if (ret) {
 457                 if (ret != -ENODEV)
 458                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 459                 return ret;
 460         }
 461
 462         if (list_empty(&dmar_drhd_units)) {
 463                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 464                 return -ENODEV;
 465         }
 466
 467 #ifdef CONFIG_DMAR
 468         if (list_empty(&dmar_rmrr_units))
 469                 printk(KERN_INFO PREFIX "No RMRR found\n");
 470 #endif
 471
 472 #ifdef CONFIG_INTR_REMAP
 473         parse_ioapics_under_ir();
 474 #endif
 475         return 0;
 476 }
 477
 478 void __init detect_intel_iommu(void)
 479 {
 480         int ret;
 481
 482         ret = dmar_table_detect();
 483
 484         {
 485 #ifdef CONFIG_INTR_REMAP
 486                 struct acpi_table_dmar *dmar;
 487                 /*
 488                  * for now we will disable dma-remapping when interrupt
 489                  * remapping is enabled.
 490                  * When support for queued invalidation for IOTLB invalidation
 491                  * is added, we will not need this any more.
 492                  */
 493                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 494                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 495                         printk(KERN_INFO
 496                                "Queued invalidation will be enabled to support "
 497                                "x2apic and Intr-remapping.\n");
 498 #endif
 499 #ifdef CONFIG_DMAR
 500                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 501                     !dmar_disabled)
 502                         iommu_detected = 1;
 503 #endif
 504         }
 505         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 506         dmar_tbl = NULL;
 507 }
 508
 509
 510 int alloc_iommu(struct dmar_drhd_unit *drhd)
 511 {
 512         struct intel_iommu *iommu;
 513         int map_size;
 514         u32 ver;
 515         static int iommu_allocated = 0;
 516         int agaw = 0;
 517
 518         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 519         if (!iommu)
 520                 return -ENOMEM;
 521
 522         iommu->seq_id = iommu_allocated++;
 523         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 524
 525         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 526         if (!iommu->reg) {
 527                 printk(KERN_ERR "IOMMU: can't map the region\n");
 528                 goto error;
 529         }
 530         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 531         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 532
 533 #ifdef CONFIG_DMAR
 534         agaw = iommu_calculate_agaw(iommu);
 535         if (agaw < 0) {
 536                 printk(KERN_ERR
 537                         "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 538                         iommu->seq_id);
 539                 goto error;
 540         }
 541 #endif
 542         iommu->agaw = agaw;
 543
 544         /* the registers might be more than one page */
 545         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 546                 cap_max_fault_reg_offset(iommu->cap));
 547         map_size = VTD_PAGE_ALIGN(map_size);
 548         if (map_size > VTD_PAGE_SIZE) {
 549                 iounmap(iommu->reg);
 550                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 551                 if (!iommu->reg) {
 552                         printk(KERN_ERR "IOMMU: can't map the region\n");
 553                         goto error;
 554                 }
 555         }
 556
 557         ver = readl(iommu->reg + DMAR_VER_REG);
 558         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 559                 (unsigned long long)drhd->reg_base_addr,
 560                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 561                 (unsigned long long)iommu->cap,
 562                 (unsigned long long)iommu->ecap);
 563
 564         spin_lock_init(&iommu->register_lock);
 565
 566         drhd->iommu = iommu;
 567         return 0;
 568 error:
 569         kfree(iommu);
 570         return -1;
 571 }
 572
 573 void free_iommu(struct intel_iommu *iommu)
 574 {
 575         if (!iommu)
 576                 return;
 577
 578 #ifdef CONFIG_DMAR
 579         free_dmar_iommu(iommu);
 580 #endif
 581
 582         if (iommu->reg)
 583                 iounmap(iommu->reg);
 584         kfree(iommu);
 585 }
 586
 587 /*
 588  * Reclaim all the submitted descriptors which have completed its work.
 589  */
 590 static inline void reclaim_free_desc(struct q_inval *qi)
 591 {
 592         while (qi->desc_status[qi->free_tail] == QI_DONE) {
 593                 qi->desc_status[qi->free_tail] = QI_FREE;
 594                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 595                 qi->free_cnt++;
 596         }
 597 }
 598
 599 static int qi_check_fault(struct intel_iommu *iommu, int index)
 600 {
 601         u32 fault;
 602         int head;
 603         struct q_inval *qi = iommu->qi;
 604         int wait_index = (index + 1) % QI_LENGTH;
 605
 606         fault = readl(iommu->reg + DMAR_FSTS_REG);
 607
 608         /*
 609          * If IQE happens, the head points to the descriptor associated
 610          * with the error. No new descriptors are fetched until the IQE
 611          * is cleared.
 612          */
 613         if (fault & DMA_FSTS_IQE) {
 614                 head = readl(iommu->reg + DMAR_IQH_REG);
 615                 if ((head >> 4) == index) {
 616                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 617                                         sizeof(struct qi_desc));
 618                         __iommu_flush_cache(iommu, &qi->desc[index],
 619                                         sizeof(struct qi_desc));
 620                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 621                         return -EINVAL;
 622                 }
 623         }
 624
 625         return 0;
 626 }
 627
 628 /*
 629  * Submit the queued invalidation descriptor to the remapping
 630  * hardware unit and wait for its completion.
 631  */
 632 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 633 {
 634         int rc = 0;
 635         struct q_inval *qi = iommu->qi;
 636         struct qi_desc *hw, wait_desc;
 637         int wait_index, index;
 638         unsigned long flags;
 639
 640         if (!qi)
 641                 return 0;
 642
 643         hw = qi->desc;
 644
 645         spin_lock_irqsave(&qi->q_lock, flags);
 646         while (qi->free_cnt < 3) {
 647                 spin_unlock_irqrestore(&qi->q_lock, flags);
 648                 cpu_relax();
 649                 spin_lock_irqsave(&qi->q_lock, flags);
 650         }
 651
 652         index = qi->free_head;
 653         wait_index = (index + 1) % QI_LENGTH;
 654
 655         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 656
 657         hw[index] = *desc;
 658
 659         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 660                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 661         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 662
 663         hw[wait_index] = wait_desc;
 664
 665         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 666         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 667
 668         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 669         qi->free_cnt -= 2;
 670
 671         /*
 672          * update the HW tail register indicating the presence of
 673          * new descriptors.
 674          */
 675         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
 676
 677         while (qi->desc_status[wait_index] != QI_DONE) {
 678                 /*
 679                  * We will leave the interrupts disabled, to prevent interrupt
 680                  * context to queue another cmd while a cmd is already submitted
 681                  * and waiting for completion on this cpu. This is to avoid
 682                  * a deadlock where the interrupt context can wait indefinitely
 683                  * for free slots in the queue.
 684                  */
 685                 rc = qi_check_fault(iommu, index);
 686                 if (rc)
 687                         goto out;
 688
 689                 spin_unlock(&qi->q_lock);
 690                 cpu_relax();
 691                 spin_lock(&qi->q_lock);
 692         }
 693 out:
 694         qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
 695
 696         reclaim_free_desc(qi);
 697         spin_unlock_irqrestore(&qi->q_lock, flags);
 698
 699         return rc;
 700 }
 701
 702 /*
 703  * Flush the global interrupt entry cache.
 704  */
 705 void qi_global_iec(struct intel_iommu *iommu)
 706 {
 707         struct qi_desc desc;
 708
 709         desc.low = QI_IEC_TYPE;
 710         desc.high = 0;
 711
 712         /* should never fail */
 713         qi_submit_sync(&desc, iommu);
 714 }
 715
 716 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 717                      u64 type, int non_present_entry_flush)
 718 {
 719         struct qi_desc desc;
 720
 721         if (non_present_entry_flush) {
 722                 if (!cap_caching_mode(iommu->cap))
 723                         return 1;
 724                 else
 725                         did = 0;
 726         }
 727
 728         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 729                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 730         desc.high = 0;
 731
 732         return qi_submit_sync(&desc, iommu);
 733 }
 734
 735 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 736                    unsigned int size_order, u64 type,
 737                    int non_present_entry_flush)
 738 {
 739         u8 dw = 0, dr = 0;
 740
 741         struct qi_desc desc;
 742         int ih = 0;
 743
 744         if (non_present_entry_flush) {
 745                 if (!cap_caching_mode(iommu->cap))
 746                         return 1;
 747                 else
 748                         did = 0;
 749         }
 750
 751         if (cap_write_drain(iommu->cap))
 752                 dw = 1;
 753
 754         if (cap_read_drain(iommu->cap))
 755                 dr = 1;
 756
 757         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 758                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 759         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 760                 | QI_IOTLB_AM(size_order);
 761
 762         return qi_submit_sync(&desc, iommu);
 763 }
 764
 765 /*
 766  * Disable Queued Invalidation interface.
 767  */
 768 void dmar_disable_qi(struct intel_iommu *iommu)
 769 {
 770         unsigned long flags;
 771         u32 sts;
 772         cycles_t start_time = get_cycles();
 773
 774         if (!ecap_qis(iommu->ecap))
 775                 return;
 776
 777         spin_lock_irqsave(&iommu->register_lock, flags);
 778
 779         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
 780         if (!(sts & DMA_GSTS_QIES))
 781                 goto end;
 782
 783         /*
 784          * Give a chance to HW to complete the pending invalidation requests.
 785          */
 786         while ((readl(iommu->reg + DMAR_IQT_REG) !=
 787                 readl(iommu->reg + DMAR_IQH_REG)) &&
 788                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
 789                 cpu_relax();
 790
 791         iommu->gcmd &= ~DMA_GCMD_QIE;
 792
 793         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 794
 795         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
 796                       !(sts & DMA_GSTS_QIES), sts);
 797 end:
 798         spin_unlock_irqrestore(&iommu->register_lock, flags);
 799 }
 800
 801 /*
 802  * Enable Queued Invalidation interface. This is a must to support
 803  * interrupt-remapping. Also used by DMA-remapping, which replaces
 804  * register based IOTLB invalidation.
 805  */
 806 int dmar_enable_qi(struct intel_iommu *iommu)
 807 {
 808         u32 cmd, sts;
 809         unsigned long flags;
 810         struct q_inval *qi;
 811
 812         if (!ecap_qis(iommu->ecap))
 813                 return -ENOENT;
 814
 815         /*
 816          * queued invalidation is already setup and enabled.
 817          */
 818         if (iommu->qi)
 819                 return 0;
 820
 821         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
 822         if (!iommu->qi)
 823                 return -ENOMEM;
 824
 825         qi = iommu->qi;
 826
 827         qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
 828         if (!qi->desc) {
 829                 kfree(qi);
 830                 iommu->qi = 0;
 831                 return -ENOMEM;
 832         }
 833
 834         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
 835         if (!qi->desc_status) {
 836                 free_page((unsigned long) qi->desc);
 837                 kfree(qi);
 838                 iommu->qi = 0;
 839                 return -ENOMEM;
 840         }
 841
 842         qi->free_head = qi->free_tail = 0;
 843         qi->free_cnt = QI_LENGTH;
 844
 845         spin_lock_init(&qi->q_lock);
 846
 847         spin_lock_irqsave(&iommu->register_lock, flags);
 848         /* write zero to the tail reg */
 849         writel(0, iommu->reg + DMAR_IQT_REG);
 850
 851         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 852
 853         cmd = iommu->gcmd | DMA_GCMD_QIE;
 854         iommu->gcmd |= DMA_GCMD_QIE;
 855         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 856
 857         /* Make sure hardware complete it */
 858         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 859         spin_unlock_irqrestore(&iommu->register_lock, flags);
 860
 861         return 0;
 862 }
 863
 864 /* iommu interrupt handling. Most stuff are MSI-like. */
 865
 866 enum faulttype {
 867         DMA_REMAP,
 868         INTR_REMAP,
 869         UNKNOWN,
 870 };
 871
 872 static const char *dma_remap_fault_reasons[] =
 873 {
 874         "Software",
 875         "Present bit in root entry is clear",
 876         "Present bit in context entry is clear",
 877         "Invalid context entry",
 878         "Access beyond MGAW",
 879         "PTE Write access is not set",
 880         "PTE Read access is not set",
 881         "Next page table ptr is invalid",
 882         "Root table address invalid",
 883         "Context table ptr is invalid",
 884         "non-zero reserved fields in RTP",
 885         "non-zero reserved fields in CTP",
 886         "non-zero reserved fields in PTE",
 887 };
 888
 889 static const char *intr_remap_fault_reasons[] =
 890 {
 891         "Detected reserved fields in the decoded interrupt-remapped request",
 892         "Interrupt index exceeded the interrupt-remapping table size",
 893         "Present field in the IRTE entry is clear",
 894         "Error accessing interrupt-remapping table pointed by IRTA_REG",
 895         "Detected reserved fields in the IRTE entry",
 896         "Blocked a compatibility format interrupt request",
 897         "Blocked an interrupt request due to source-id verification failure",
 898 };
 899
 900 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
 901
 902 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
 903 {
 904         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
 905                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
 906                 *fault_type = INTR_REMAP;
 907                 return intr_remap_fault_reasons[fault_reason - 0x20];
 908         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
 909                 *fault_type = DMA_REMAP;
 910                 return dma_remap_fault_reasons[fault_reason];
 911         } else {
 912                 *fault_type = UNKNOWN;
 913                 return "Unknown";
 914         }
 915 }
 916
 917 void dmar_msi_unmask(unsigned int irq)
 918 {
 919         struct intel_iommu *iommu = get_irq_data(irq);
 920         unsigned long flag;
 921
 922         /* unmask it */
 923         spin_lock_irqsave(&iommu->register_lock, flag);
 924         writel(0, iommu->reg + DMAR_FECTL_REG);
 925         /* Read a reg to force flush the post write */
 926         readl(iommu->reg + DMAR_FECTL_REG);
 927         spin_unlock_irqrestore(&iommu->register_lock, flag);
 928 }
 929
 930 void dmar_msi_mask(unsigned int irq)
 931 {
 932         unsigned long flag;
 933         struct intel_iommu *iommu = get_irq_data(irq);
 934
 935         /* mask it */
 936         spin_lock_irqsave(&iommu->register_lock, flag);
 937         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
 938         /* Read a reg to force flush the post write */
 939         readl(iommu->reg + DMAR_FECTL_REG);
 940         spin_unlock_irqrestore(&iommu->register_lock, flag);
 941 }
 942
 943 void dmar_msi_write(int irq, struct msi_msg *msg)
 944 {
 945         struct intel_iommu *iommu = get_irq_data(irq);
 946         unsigned long flag;
 947
 948         spin_lock_irqsave(&iommu->register_lock, flag);
 949         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
 950         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
 951         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
 952         spin_unlock_irqrestore(&iommu->register_lock, flag);
 953 }
 954
 955 void dmar_msi_read(int irq, struct msi_msg *msg)
 956 {
 957         struct intel_iommu *iommu = get_irq_data(irq);
 958         unsigned long flag;
 959
 960         spin_lock_irqsave(&iommu->register_lock, flag);
 961         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
 962         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
 963         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
 964         spin_unlock_irqrestore(&iommu->register_lock, flag);
 965 }
 966
 967 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
 968                 u8 fault_reason, u16 source_id, unsigned long long addr)
 969 {
 970         const char *reason;
 971         int fault_type;
 972
 973         reason = dmar_get_fault_reason(fault_reason, &fault_type);
 974
 975         if (fault_type == INTR_REMAP)
 976                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
 977                        "fault index %llx\n"
 978                         "INTR-REMAP:[fault reason %02d] %s\n",
 979                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 980                         PCI_FUNC(source_id & 0xFF), addr >> 48,
 981                         fault_reason, reason);
 982         else
 983                 printk(KERN_ERR
 984                        "DMAR:[%s] Request device [%02x:%02x.%d] "
 985                        "fault addr %llx \n"
 986                        "DMAR:[fault reason %02d] %s\n",
 987                        (type ? "DMA Read" : "DMA Write"),
 988                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 989                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
 990         return 0;
 991 }
 992
 993 #define PRIMARY_FAULT_REG_LEN (16)
 994 irqreturn_t dmar_fault(int irq, void *dev_id)
 995 {
 996         struct intel_iommu *iommu = dev_id;
 997         int reg, fault_index;
 998         u32 fault_status;
 999         unsigned long flag;
1000
1001         spin_lock_irqsave(&iommu->register_lock, flag);
1002         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1003         if (fault_status)
1004                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1005                        fault_status);
1006
1007         /* TBD: ignore advanced fault log currently */
1008         if (!(fault_status & DMA_FSTS_PPF))
1009                 goto clear_rest;
1010
1011         fault_index = dma_fsts_fault_record_index(fault_status);
1012         reg = cap_fault_reg_offset(iommu->cap);
1013         while (1) {
1014                 u8 fault_reason;
1015                 u16 source_id;
1016                 u64 guest_addr;
1017                 int type;
1018                 u32 data;
1019
1020                 /* highest 32 bits */
1021                 data = readl(iommu->reg + reg +
1022                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1023                 if (!(data & DMA_FRCD_F))
1024                         break;
1025
1026                 fault_reason = dma_frcd_fault_reason(data);
1027                 type = dma_frcd_type(data);
1028
1029                 data = readl(iommu->reg + reg +
1030                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1031                 source_id = dma_frcd_source_id(data);
1032
1033                 guest_addr = dmar_readq(iommu->reg + reg +
1034                                 fault_index * PRIMARY_FAULT_REG_LEN);
1035                 guest_addr = dma_frcd_page_addr(guest_addr);
1036                 /* clear the fault */
1037                 writel(DMA_FRCD_F, iommu->reg + reg +
1038                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1039
1040                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1041
1042                 dmar_fault_do_one(iommu, type, fault_reason,
1043                                 source_id, guest_addr);
1044
1045                 fault_index++;
1046                 if (fault_index > cap_num_fault_regs(iommu->cap))
1047                         fault_index = 0;
1048                 spin_lock_irqsave(&iommu->register_lock, flag);
1049         }
1050 clear_rest:
1051         /* clear all the other faults */
1052         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1053         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1054
1055         spin_unlock_irqrestore(&iommu->register_lock, flag);
1056         return IRQ_HANDLED;
1057 }
1058
1059 int dmar_set_interrupt(struct intel_iommu *iommu)
1060 {
1061         int irq, ret;
1062
1063         /*
1064          * Check if the fault interrupt is already initialized.
1065          */
1066         if (iommu->irq)
1067                 return 0;
1068
1069         irq = create_irq();
1070         if (!irq) {
1071                 printk(KERN_ERR "IOMMU: no free vectors\n");
1072                 return -EINVAL;
1073         }
1074
1075         set_irq_data(irq, iommu);
1076         iommu->irq = irq;
1077
1078         ret = arch_setup_dmar_msi(irq);
1079         if (ret) {
1080                 set_irq_data(irq, NULL);
1081                 iommu->irq = 0;
1082                 destroy_irq(irq);
1083                 return 0;
1084         }
1085
1086         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1087         if (ret)
1088                 printk(KERN_ERR "IOMMU: can't request irq\n");
1089         return ret;
1090 }
1091
1092 int __init enable_drhd_fault_handling(void)
1093 {
1094         struct dmar_drhd_unit *drhd;
1095
1096         /*
1097          * Enable fault control interrupt.
1098          */
1099         for_each_drhd_unit(drhd) {
1100                 int ret;
1101                 struct intel_iommu *iommu = drhd->iommu;
1102                 ret = dmar_set_interrupt(iommu);
1103
1104                 if (ret) {
1105                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1106                                " interrupt, ret %d\n",
1107                                (unsigned long long)drhd->reg_base_addr, ret);
1108                         return -1;
1109                 }
1110         }
1111
1112         return 0;
1113 }