drivers/dma/ioatdma.c

   1 /*
   2  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License as published by the Free
   6  * Software Foundation; either version 2 of the License, or (at your option)
   7  * any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc., 59
  16  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in the
  19  * file called COPYING.
  20  */
  21
  22 /*
  23  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  24  * copy operations.
  25  */
  26
  27 #include <linux/init.h>
  28 #include <linux/module.h>
  29 #include <linux/pci.h>
  30 #include <linux/interrupt.h>
  31 #include <linux/dmaengine.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-mapping.h>
  34 #include "ioatdma.h"
  35 #include "ioatdma_io.h"
  36 #include "ioatdma_registers.h"
  37 #include "ioatdma_hw.h"
  38
  39 #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
  40 #define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
  41 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
  42
  43 /* internal functions */
  44 static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
  45 static void __devexit ioat_remove(struct pci_dev *pdev);
  46
  47 static int enumerate_dma_channels(struct ioat_device *device)
  48 {
  49         u8 xfercap_scale;
  50         u32 xfercap;
  51         int i;
  52         struct ioat_dma_chan *ioat_chan;
  53
  54         device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET);
  55         xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET);
  56         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
  57
  58         for (i = 0; i < device->common.chancnt; i++) {
  59                 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
  60                 if (!ioat_chan) {
  61                         device->common.chancnt = i;
  62                         break;
  63                 }
  64
  65                 ioat_chan->device = device;
  66                 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
  67                 ioat_chan->xfercap = xfercap;
  68                 spin_lock_init(&ioat_chan->cleanup_lock);
  69                 spin_lock_init(&ioat_chan->desc_lock);
  70                 INIT_LIST_HEAD(&ioat_chan->free_desc);
  71                 INIT_LIST_HEAD(&ioat_chan->used_desc);
  72                 /* This should be made common somewhere in dmaengine.c */
  73                 ioat_chan->common.device = &device->common;
  74                 ioat_chan->common.client = NULL;
  75                 list_add_tail(&ioat_chan->common.device_node,
  76                               &device->common.channels);
  77         }
  78         return device->common.chancnt;
  79 }
  80
  81 static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
  82         struct ioat_dma_chan *ioat_chan,
  83         int flags)
  84 {
  85         struct ioat_dma_descriptor *desc;
  86         struct ioat_desc_sw *desc_sw;
  87         struct ioat_device *ioat_device;
  88         dma_addr_t phys;
  89
  90         ioat_device = to_ioat_device(ioat_chan->common.device);
  91         desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
  92         if (unlikely(!desc))
  93                 return NULL;
  94
  95         desc_sw = kzalloc(sizeof(*desc_sw), flags);
  96         if (unlikely(!desc_sw)) {
  97                 pci_pool_free(ioat_device->dma_pool, desc, phys);
  98                 return NULL;
  99         }
 100
 101         memset(desc, 0, sizeof(*desc));
 102         desc_sw->hw = desc;
 103         desc_sw->phys = phys;
 104
 105         return desc_sw;
 106 }
 107
 108 #define INITIAL_IOAT_DESC_COUNT 128
 109
 110 static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
 111
 112 /* returns the actual number of allocated descriptors */
 113 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
 114 {
 115         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 116         struct ioat_desc_sw *desc = NULL;
 117         u16 chanctrl;
 118         u32 chanerr;
 119         int i;
 120         LIST_HEAD(tmp_list);
 121
 122         /*
 123          * In-use bit automatically set by reading chanctrl
 124          * If 0, we got it, if 1, someone else did
 125          */
 126         chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
 127         if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
 128                 return -EBUSY;
 129
 130         /* Setup register to interrupt and write completion status on error */
 131         chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
 132                 IOAT_CHANCTRL_ERR_INT_EN |
 133                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 134                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 135         ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
 136
 137         chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET);
 138         if (chanerr) {
 139                 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
 140                 ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr);
 141         }
 142
 143         /* Allocate descriptors */
 144         for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
 145                 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
 146                 if (!desc) {
 147                         printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
 148                         break;
 149                 }
 150                 list_add_tail(&desc->node, &tmp_list);
 151         }
 152         spin_lock_bh(&ioat_chan->desc_lock);
 153         list_splice(&tmp_list, &ioat_chan->free_desc);
 154         spin_unlock_bh(&ioat_chan->desc_lock);
 155
 156         /* allocate a completion writeback area */
 157         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 158         ioat_chan->completion_virt =
 159                 pci_pool_alloc(ioat_chan->device->completion_pool,
 160                                GFP_KERNEL,
 161                                &ioat_chan->completion_addr);
 162         memset(ioat_chan->completion_virt, 0,
 163                sizeof(*ioat_chan->completion_virt));
 164         ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW,
 165                        ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF);
 166         ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH,
 167                        ((u64) ioat_chan->completion_addr) >> 32);
 168
 169         ioat_start_null_desc(ioat_chan);
 170         return i;
 171 }
 172
 173 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
 174
 175 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
 176 {
 177         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 178         struct ioat_device *ioat_device = to_ioat_device(chan->device);
 179         struct ioat_desc_sw *desc, *_desc;
 180         u16 chanctrl;
 181         int in_use_descs = 0;
 182
 183         ioat_dma_memcpy_cleanup(ioat_chan);
 184
 185         ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
 186
 187         spin_lock_bh(&ioat_chan->desc_lock);
 188         list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
 189                 in_use_descs++;
 190                 list_del(&desc->node);
 191                 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
 192                 kfree(desc);
 193         }
 194         list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
 195                 list_del(&desc->node);
 196                 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
 197                 kfree(desc);
 198         }
 199         spin_unlock_bh(&ioat_chan->desc_lock);
 200
 201         pci_pool_free(ioat_device->completion_pool,
 202                       ioat_chan->completion_virt,
 203                       ioat_chan->completion_addr);
 204
 205         /* one is ok since we left it on there on purpose */
 206         if (in_use_descs > 1)
 207                 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
 208                         in_use_descs - 1);
 209
 210         ioat_chan->last_completion = ioat_chan->completion_addr = 0;
 211
 212         /* Tell hw the chan is free */
 213         chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
 214         chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
 215         ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
 216 }
 217
 218 /**
 219  * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
 220  * @chan: IOAT DMA channel handle
 221  * @dest: DMA destination address
 222  * @src: DMA source address
 223  * @len: transaction length in bytes
 224  */
 225
 226 static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
 227                                        dma_addr_t dest,
 228                                        dma_addr_t src,
 229                                        size_t len)
 230 {
 231         struct ioat_desc_sw *first;
 232         struct ioat_desc_sw *prev;
 233         struct ioat_desc_sw *new;
 234         dma_cookie_t cookie;
 235         LIST_HEAD(new_chain);
 236         u32 copy;
 237         size_t orig_len;
 238         dma_addr_t orig_src, orig_dst;
 239         unsigned int desc_count = 0;
 240         unsigned int append = 0;
 241
 242         if (!ioat_chan || !dest || !src)
 243                 return -EFAULT;
 244
 245         if (!len)
 246                 return ioat_chan->common.cookie;
 247
 248         orig_len = len;
 249         orig_src = src;
 250         orig_dst = dest;
 251
 252         first = NULL;
 253         prev = NULL;
 254
 255         spin_lock_bh(&ioat_chan->desc_lock);
 256
 257         while (len) {
 258                 if (!list_empty(&ioat_chan->free_desc)) {
 259                         new = to_ioat_desc(ioat_chan->free_desc.next);
 260                         list_del(&new->node);
 261                 } else {
 262                         /* try to get another desc */
 263                         new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 264                         /* will this ever happen? */
 265                         /* TODO add upper limit on these */
 266                         BUG_ON(!new);
 267                 }
 268
 269                 copy = min((u32) len, ioat_chan->xfercap);
 270
 271                 new->hw->size = copy;
 272                 new->hw->ctl = 0;
 273                 new->hw->src_addr = src;
 274                 new->hw->dst_addr = dest;
 275                 new->cookie = 0;
 276
 277                 /* chain together the physical address list for the HW */
 278                 if (!first)
 279                         first = new;
 280                 else
 281                         prev->hw->next = (u64) new->phys;
 282
 283                 prev = new;
 284
 285                 len  -= copy;
 286                 dest += copy;
 287                 src  += copy;
 288
 289                 list_add_tail(&new->node, &new_chain);
 290                 desc_count++;
 291         }
 292         new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 293         new->hw->next = 0;
 294
 295         /* cookie incr and addition to used_list must be atomic */
 296
 297         cookie = ioat_chan->common.cookie;
 298         cookie++;
 299         if (cookie < 0)
 300                 cookie = 1;
 301         ioat_chan->common.cookie = new->cookie = cookie;
 302
 303         pci_unmap_addr_set(new, src, orig_src);
 304         pci_unmap_addr_set(new, dst, orig_dst);
 305         pci_unmap_len_set(new, src_len, orig_len);
 306         pci_unmap_len_set(new, dst_len, orig_len);
 307
 308         /* write address into NextDescriptor field of last desc in chain */
 309         to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
 310         list_splice_init(&new_chain, ioat_chan->used_desc.prev);
 311
 312         ioat_chan->pending += desc_count;
 313         if (ioat_chan->pending >= 20) {
 314                 append = 1;
 315                 ioat_chan->pending = 0;
 316         }
 317
 318         spin_unlock_bh(&ioat_chan->desc_lock);
 319
 320         if (append)
 321                 ioatdma_chan_write8(ioat_chan,
 322                                     IOAT_CHANCMD_OFFSET,
 323                                     IOAT_CHANCMD_APPEND);
 324         return cookie;
 325 }
 326
 327 /**
 328  * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
 329  * @chan: IOAT DMA channel handle
 330  * @dest: DMA destination address
 331  * @src: DMA source address
 332  * @len: transaction length in bytes
 333  */
 334
 335 static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
 336                                                void *dest,
 337                                                void *src,
 338                                                size_t len)
 339 {
 340         dma_addr_t dest_addr;
 341         dma_addr_t src_addr;
 342         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 343
 344         dest_addr = pci_map_single(ioat_chan->device->pdev,
 345                 dest, len, PCI_DMA_FROMDEVICE);
 346         src_addr = pci_map_single(ioat_chan->device->pdev,
 347                 src, len, PCI_DMA_TODEVICE);
 348
 349         return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
 350 }
 351
 352 /**
 353  * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
 354  * @chan: IOAT DMA channel handle
 355  * @page: pointer to the page to copy to
 356  * @offset: offset into that page
 357  * @src: DMA source address
 358  * @len: transaction length in bytes
 359  */
 360
 361 static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
 362                                               struct page *page,
 363                                               unsigned int offset,
 364                                               void *src,
 365                                               size_t len)
 366 {
 367         dma_addr_t dest_addr;
 368         dma_addr_t src_addr;
 369         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 370
 371         dest_addr = pci_map_page(ioat_chan->device->pdev,
 372                 page, offset, len, PCI_DMA_FROMDEVICE);
 373         src_addr = pci_map_single(ioat_chan->device->pdev,
 374                 src, len, PCI_DMA_TODEVICE);
 375
 376         return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
 377 }
 378
 379 /**
 380  * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
 381  * @chan: IOAT DMA channel handle
 382  * @dest_pg: pointer to the page to copy to
 383  * @dest_off: offset into that page
 384  * @src_pg: pointer to the page to copy from
 385  * @src_off: offset into that page
 386  * @len: transaction length in bytes. This is guaranteed to not make a copy
 387  *       across a page boundary.
 388  */
 389
 390 static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
 391                                              struct page *dest_pg,
 392                                              unsigned int dest_off,
 393                                              struct page *src_pg,
 394                                              unsigned int src_off,
 395                                              size_t len)
 396 {
 397         dma_addr_t dest_addr;
 398         dma_addr_t src_addr;
 399         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 400
 401         dest_addr = pci_map_page(ioat_chan->device->pdev,
 402                 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
 403         src_addr = pci_map_page(ioat_chan->device->pdev,
 404                 src_pg, src_off, len, PCI_DMA_TODEVICE);
 405
 406         return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
 407 }
 408
 409 /**
 410  * ioat_dma_memcpy_issue_pending - push potentially unrecognoized appended descriptors to hw
 411  * @chan: DMA channel handle
 412  */
 413
 414 static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
 415 {
 416         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 417
 418         if (ioat_chan->pending != 0) {
 419                 ioat_chan->pending = 0;
 420                 ioatdma_chan_write8(ioat_chan,
 421                                     IOAT_CHANCMD_OFFSET,
 422                                     IOAT_CHANCMD_APPEND);
 423         }
 424 }
 425
 426 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
 427 {
 428         unsigned long phys_complete;
 429         struct ioat_desc_sw *desc, *_desc;
 430         dma_cookie_t cookie = 0;
 431
 432         prefetch(chan->completion_virt);
 433
 434         if (!spin_trylock(&chan->cleanup_lock))
 435                 return;
 436
 437         /* The completion writeback can happen at any time,
 438            so reads by the driver need to be atomic operations
 439            The descriptor physical addresses are limited to 32-bits
 440            when the CPU can only do a 32-bit mov */
 441
 442 #if (BITS_PER_LONG == 64)
 443         phys_complete =
 444         chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 445 #else
 446         phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
 447 #endif
 448
 449         if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
 450                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
 451                 printk("IOAT: Channel halted, chanerr = %x\n",
 452                         ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET));
 453
 454                 /* TODO do something to salvage the situation */
 455         }
 456
 457         if (phys_complete == chan->last_completion) {
 458                 spin_unlock(&chan->cleanup_lock);
 459                 return;
 460         }
 461
 462         spin_lock_bh(&chan->desc_lock);
 463         list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
 464
 465                 /*
 466                  * Incoming DMA requests may use multiple descriptors, due to
 467                  * exceeding xfercap, perhaps. If so, only the last one will
 468                  * have a cookie, and require unmapping.
 469                  */
 470                 if (desc->cookie) {
 471                         cookie = desc->cookie;
 472
 473                         /* yes we are unmapping both _page and _single alloc'd
 474                            regions with unmap_page. Is this *really* that bad?
 475                         */
 476                         pci_unmap_page(chan->device->pdev,
 477                                         pci_unmap_addr(desc, dst),
 478                                         pci_unmap_len(desc, dst_len),
 479                                         PCI_DMA_FROMDEVICE);
 480                         pci_unmap_page(chan->device->pdev,
 481                                         pci_unmap_addr(desc, src),
 482                                         pci_unmap_len(desc, src_len),
 483                                         PCI_DMA_TODEVICE);
 484                 }
 485
 486                 if (desc->phys != phys_complete) {
 487                         /* a completed entry, but not the last, so cleanup */
 488                         list_del(&desc->node);
 489                         list_add_tail(&desc->node, &chan->free_desc);
 490                 } else {
 491                         /* last used desc. Do not remove, so we can append from
 492                            it, but don't look at it next time, either */
 493                         desc->cookie = 0;
 494
 495                         /* TODO check status bits? */
 496                         break;
 497                 }
 498         }
 499
 500         spin_unlock_bh(&chan->desc_lock);
 501
 502         chan->last_completion = phys_complete;
 503         if (cookie != 0)
 504                 chan->completed_cookie = cookie;
 505
 506         spin_unlock(&chan->cleanup_lock);
 507 }
 508
 509 /**
 510  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
 511  * @chan: IOAT DMA channel handle
 512  * @cookie: DMA transaction identifier
 513  */
 514
 515 static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
 516                                             dma_cookie_t cookie,
 517                                             dma_cookie_t *done,
 518                                             dma_cookie_t *used)
 519 {
 520         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 521         dma_cookie_t last_used;
 522         dma_cookie_t last_complete;
 523         enum dma_status ret;
 524
 525         last_used = chan->cookie;
 526         last_complete = ioat_chan->completed_cookie;
 527
 528         if (done)
 529                 *done= last_complete;
 530         if (used)
 531                 *used = last_used;
 532
 533         ret = dma_async_is_complete(cookie, last_complete, last_used);
 534         if (ret == DMA_SUCCESS)
 535                 return ret;
 536
 537         ioat_dma_memcpy_cleanup(ioat_chan);
 538
 539         last_used = chan->cookie;
 540         last_complete = ioat_chan->completed_cookie;
 541
 542         if (done)
 543                 *done= last_complete;
 544         if (used)
 545                 *used = last_used;
 546
 547         return dma_async_is_complete(cookie, last_complete, last_used);
 548 }
 549
 550 /* PCI API */
 551
 552 static struct pci_device_id ioat_pci_tbl[] = {
 553         { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
 554         { 0, }
 555 };
 556
 557 static struct pci_driver ioat_pci_drv = {
 558         .name   = "ioatdma",
 559         .id_table = ioat_pci_tbl,
 560         .probe  = ioat_probe,
 561         .remove = __devexit_p(ioat_remove),
 562 };
 563
 564 static irqreturn_t ioat_do_interrupt(int irq, void *data, struct pt_regs *regs)
 565 {
 566         struct ioat_device *instance = data;
 567         unsigned long attnstatus;
 568         u8 intrctrl;
 569
 570         intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET);
 571
 572         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
 573                 return IRQ_NONE;
 574
 575         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
 576                 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
 577                 return IRQ_NONE;
 578         }
 579
 580         attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET);
 581
 582         printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
 583
 584         ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
 585         return IRQ_HANDLED;
 586 }
 587
 588 static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
 589 {
 590         struct ioat_desc_sw *desc;
 591
 592         spin_lock_bh(&ioat_chan->desc_lock);
 593
 594         if (!list_empty(&ioat_chan->free_desc)) {
 595                 desc = to_ioat_desc(ioat_chan->free_desc.next);
 596                 list_del(&desc->node);
 597         } else {
 598                 /* try to get another desc */
 599                 spin_unlock_bh(&ioat_chan->desc_lock);
 600                 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
 601                 spin_lock_bh(&ioat_chan->desc_lock);
 602                 /* will this ever happen? */
 603                 BUG_ON(!desc);
 604         }
 605
 606         desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
 607         desc->hw->next = 0;
 608
 609         list_add_tail(&desc->node, &ioat_chan->used_desc);
 610         spin_unlock_bh(&ioat_chan->desc_lock);
 611
 612 #if (BITS_PER_LONG == 64)
 613         ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys);
 614 #else
 615         ioatdma_chan_write32(ioat_chan,
 616                              IOAT_CHAINADDR_OFFSET_LOW,
 617                              (u32) desc->phys);
 618         ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
 619 #endif
 620         ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
 621 }
 622
 623 /*
 624  * Perform a IOAT transaction to verify the HW works.
 625  */
 626 #define IOAT_TEST_SIZE 2000
 627
 628 static int ioat_self_test(struct ioat_device *device)
 629 {
 630         int i;
 631         u8 *src;
 632         u8 *dest;
 633         struct dma_chan *dma_chan;
 634         dma_cookie_t cookie;
 635         int err = 0;
 636
 637         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
 638         if (!src)
 639                 return -ENOMEM;
 640         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
 641         if (!dest) {
 642                 kfree(src);
 643                 return -ENOMEM;
 644         }
 645
 646         /* Fill in src buffer */
 647         for (i = 0; i < IOAT_TEST_SIZE; i++)
 648                 src[i] = (u8)i;
 649
 650         /* Start copy, using first DMA channel */
 651         dma_chan = container_of(device->common.channels.next,
 652                                 struct dma_chan,
 653                                 device_node);
 654         if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
 655                 err = -ENODEV;
 656                 goto out;
 657         }
 658
 659         cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
 660         ioat_dma_memcpy_issue_pending(dma_chan);
 661         msleep(1);
 662
 663         if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
 664                 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
 665                 err = -ENODEV;
 666                 goto free_resources;
 667         }
 668         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
 669                 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
 670                 err = -ENODEV;
 671                 goto free_resources;
 672         }
 673
 674 free_resources:
 675         ioat_dma_free_chan_resources(dma_chan);
 676 out:
 677         kfree(src);
 678         kfree(dest);
 679         return err;
 680 }
 681
 682 static int __devinit ioat_probe(struct pci_dev *pdev,
 683                                 const struct pci_device_id *ent)
 684 {
 685         int err;
 686         unsigned long mmio_start, mmio_len;
 687         void *reg_base;
 688         struct ioat_device *device;
 689
 690         err = pci_enable_device(pdev);
 691         if (err)
 692                 goto err_enable_device;
 693
 694         err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
 695         if (err)
 696                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
 697         if (err)
 698                 goto err_set_dma_mask;
 699
 700         err = pci_request_regions(pdev, ioat_pci_drv.name);
 701         if (err)
 702                 goto err_request_regions;
 703
 704         mmio_start = pci_resource_start(pdev, 0);
 705         mmio_len = pci_resource_len(pdev, 0);
 706
 707         reg_base = ioremap(mmio_start, mmio_len);
 708         if (!reg_base) {
 709                 err = -ENOMEM;
 710                 goto err_ioremap;
 711         }
 712
 713         device = kzalloc(sizeof(*device), GFP_KERNEL);
 714         if (!device) {
 715                 err = -ENOMEM;
 716                 goto err_kzalloc;
 717         }
 718
 719         /* DMA coherent memory pool for DMA descriptor allocations */
 720         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
 721                 sizeof(struct ioat_dma_descriptor), 64, 0);
 722         if (!device->dma_pool) {
 723                 err = -ENOMEM;
 724                 goto err_dma_pool;
 725         }
 726
 727         device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
 728         if (!device->completion_pool) {
 729                 err = -ENOMEM;
 730                 goto err_completion_pool;
 731         }
 732
 733         device->pdev = pdev;
 734         pci_set_drvdata(pdev, device);
 735 #ifdef CONFIG_PCI_MSI
 736         if (pci_enable_msi(pdev) == 0) {
 737                 device->msi = 1;
 738         } else {
 739                 device->msi = 0;
 740         }
 741 #endif
 742         err = request_irq(pdev->irq, &ioat_do_interrupt, SA_SHIRQ, "ioat",
 743                 device);
 744         if (err)
 745                 goto err_irq;
 746
 747         device->reg_base = reg_base;
 748
 749         ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN);
 750         pci_set_master(pdev);
 751
 752         INIT_LIST_HEAD(&device->common.channels);
 753         enumerate_dma_channels(device);
 754
 755         device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
 756         device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
 757         device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
 758         device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
 759         device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
 760         device->common.device_memcpy_complete = ioat_dma_is_complete;
 761         device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
 762         printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
 763                 device->common.chancnt);
 764
 765         err = ioat_self_test(device);
 766         if (err)
 767                 goto err_self_test;
 768
 769         dma_async_device_register(&device->common);
 770
 771         return 0;
 772
 773 err_self_test:
 774 err_irq:
 775         pci_pool_destroy(device->completion_pool);
 776 err_completion_pool:
 777         pci_pool_destroy(device->dma_pool);
 778 err_dma_pool:
 779         kfree(device);
 780 err_kzalloc:
 781         iounmap(reg_base);
 782 err_ioremap:
 783         pci_release_regions(pdev);
 784 err_request_regions:
 785 err_set_dma_mask:
 786         pci_disable_device(pdev);
 787 err_enable_device:
 788         return err;
 789 }
 790
 791 static void __devexit ioat_remove(struct pci_dev *pdev)
 792 {
 793         struct ioat_device *device;
 794         struct dma_chan *chan, *_chan;
 795         struct ioat_dma_chan *ioat_chan;
 796
 797         device = pci_get_drvdata(pdev);
 798         dma_async_device_unregister(&device->common);
 799
 800         free_irq(device->pdev->irq, device);
 801 #ifdef CONFIG_PCI_MSI
 802         if (device->msi)
 803                 pci_disable_msi(device->pdev);
 804 #endif
 805         pci_pool_destroy(device->dma_pool);
 806         pci_pool_destroy(device->completion_pool);
 807         iounmap(device->reg_base);
 808         pci_release_regions(pdev);
 809         pci_disable_device(pdev);
 810         list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
 811                 ioat_chan = to_ioat_chan(chan);
 812                 list_del(&chan->device_node);
 813                 kfree(ioat_chan);
 814         }
 815         kfree(device);
 816 }
 817
 818 /* MODULE API */
 819 MODULE_VERSION("1.7");
 820 MODULE_LICENSE("GPL");
 821 MODULE_AUTHOR("Intel Corporation");
 822
 823 static int __init ioat_init_module(void)
 824 {
 825         /* it's currently unsafe to unload this module */
 826         /* if forced, worst case is that rmmod hangs */
 827         if (THIS_MODULE != NULL)
 828                 THIS_MODULE->unsafe = 1;
 829
 830         return pci_module_init(&ioat_pci_drv);
 831 }
 832
 833 module_init(ioat_init_module);
 834
 835 static void __exit ioat_exit_module(void)
 836 {
 837         pci_unregister_driver(&ioat_pci_drv);
 838 }
 839
 840 module_exit(ioat_exit_module);