Merge remote-tracking branch 'moduleh/module.h-split'

[linux-2.6/next.git] / drivers / infiniband / hw / ipath / ipath_sdma.c

/*
 * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/spinlock.h>
#include <linux/gfp.h>

#include "ipath_kernel.h"
#include "ipath_verbs.h"
#include "ipath_common.h"

#define SDMA_DESCQ_SZ PAGE_SIZE /* 256 entries per 4KB page */

static void vl15_watchdog_enq(struct ipath_devdata *dd)
{
        /* ipath_sdma_lock must already be held */
        if (atomic_inc_return(&dd->ipath_sdma_vl15_count) == 1) {
                unsigned long interval = (HZ + 19) / 20;
                dd->ipath_sdma_vl15_timer.expires = jiffies + interval;
                add_timer(&dd->ipath_sdma_vl15_timer);
        }
}

static void vl15_watchdog_deq(struct ipath_devdata *dd)
{
        /* ipath_sdma_lock must already be held */
        if (atomic_dec_return(&dd->ipath_sdma_vl15_count) != 0) {
                unsigned long interval = (HZ + 19) / 20;
                mod_timer(&dd->ipath_sdma_vl15_timer, jiffies + interval);
        } else {
                del_timer(&dd->ipath_sdma_vl15_timer);
        }
}

static void vl15_watchdog_timeout(unsigned long opaque)
{
        struct ipath_devdata *dd = (struct ipath_devdata *)opaque;

        if (atomic_read(&dd->ipath_sdma_vl15_count) != 0) {
                ipath_dbg("vl15 watchdog timeout - clearing\n");
                ipath_cancel_sends(dd, 1);
                ipath_hol_down(dd);
        } else {
                ipath_dbg("vl15 watchdog timeout - "
                          "condition already cleared\n");
        }
}

static void unmap_desc(struct ipath_devdata *dd, unsigned head)
{
        __le64 *descqp = &dd->ipath_sdma_descq[head].qw[0];
        u64 desc[2];
        dma_addr_t addr;
        size_t len;

        desc[0] = le64_to_cpu(descqp[0]);
        desc[1] = le64_to_cpu(descqp[1]);

        addr = (desc[1] << 32) | (desc[0] >> 32);
        len = (desc[0] >> 14) & (0x7ffULL << 2);
        dma_unmap_single(&dd->pcidev->dev, addr, len, DMA_TO_DEVICE);
}

/*
 * ipath_sdma_lock should be locked before calling this.
 */
int ipath_sdma_make_progress(struct ipath_devdata *dd)
{
        struct list_head *lp = NULL;
        struct ipath_sdma_txreq *txp = NULL;
        u16 dmahead;
        u16 start_idx = 0;
        int progress = 0;

        if (!list_empty(&dd->ipath_sdma_activelist)) {
                lp = dd->ipath_sdma_activelist.next;
                txp = list_entry(lp, struct ipath_sdma_txreq, list);
                start_idx = txp->start_idx;
        }

        /*
         * Read the SDMA head register in order to know that the
         * interrupt clear has been written to the chip.
         * Otherwise, we may not get an interrupt for the last
         * descriptor in the queue.
         */
        dmahead = (u16)ipath_read_kreg32(dd, dd->ipath_kregs->kr_senddmahead);
        /* sanity check return value for error handling (chip reset, etc.) */
        if (dmahead >= dd->ipath_sdma_descq_cnt)
                goto done;

        while (dd->ipath_sdma_descq_head != dmahead) {
                if (txp && txp->flags & IPATH_SDMA_TXREQ_F_FREEDESC &&
                    dd->ipath_sdma_descq_head == start_idx) {
                        unmap_desc(dd, dd->ipath_sdma_descq_head);
                        start_idx++;
                        if (start_idx == dd->ipath_sdma_descq_cnt)
                                start_idx = 0;
                }

                /* increment free count and head */
                dd->ipath_sdma_descq_removed++;
                if (++dd->ipath_sdma_descq_head == dd->ipath_sdma_descq_cnt)
                        dd->ipath_sdma_descq_head = 0;

                if (txp && txp->next_descq_idx == dd->ipath_sdma_descq_head) {
                        /* move to notify list */
                        if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
                                vl15_watchdog_deq(dd);
                        list_move_tail(lp, &dd->ipath_sdma_notifylist);
                        if (!list_empty(&dd->ipath_sdma_activelist)) {
                                lp = dd->ipath_sdma_activelist.next;
                                txp = list_entry(lp, struct ipath_sdma_txreq,
                                                 list);
                                start_idx = txp->start_idx;
                        } else {
                                lp = NULL;
                                txp = NULL;
                        }
                }
                progress = 1;
        }

        if (progress)
                tasklet_hi_schedule(&dd->ipath_sdma_notify_task);

done:
        return progress;
}

static void ipath_sdma_notify(struct ipath_devdata *dd, struct list_head *list)
{
        struct ipath_sdma_txreq *txp, *txp_next;

        list_for_each_entry_safe(txp, txp_next, list, list) {
                list_del_init(&txp->list);

                if (txp->callback)
                        (*txp->callback)(txp->callback_cookie,
                                         txp->callback_status);
        }
}

static void sdma_notify_taskbody(struct ipath_devdata *dd)
{
        unsigned long flags;
        struct list_head list;

        INIT_LIST_HEAD(&list);

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);

        list_splice_init(&dd->ipath_sdma_notifylist, &list);

        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);

        ipath_sdma_notify(dd, &list);

        /*
         * The IB verbs layer needs to see the callback before getting
         * the call to ipath_ib_piobufavail() because the callback
         * handles releasing resources the next send will need.
         * Otherwise, we could do these calls in
         * ipath_sdma_make_progress().
         */
        ipath_ib_piobufavail(dd->verbs_dev);
}

static void sdma_notify_task(unsigned long opaque)
{
        struct ipath_devdata *dd = (struct ipath_devdata *)opaque;

        if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
                sdma_notify_taskbody(dd);
}

static void dump_sdma_state(struct ipath_devdata *dd)
{
        unsigned long reg;

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmastatus);
        ipath_cdbg(VERBOSE, "kr_senddmastatus: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendctrl);
        ipath_cdbg(VERBOSE, "kr_sendctrl: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask0);
        ipath_cdbg(VERBOSE, "kr_senddmabufmask0: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask1);
        ipath_cdbg(VERBOSE, "kr_senddmabufmask1: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask2);
        ipath_cdbg(VERBOSE, "kr_senddmabufmask2: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmatail);
        ipath_cdbg(VERBOSE, "kr_senddmatail: 0x%016lx\n", reg);

        reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmahead);
        ipath_cdbg(VERBOSE, "kr_senddmahead: 0x%016lx\n", reg);
}

static void sdma_abort_task(unsigned long opaque)
{
        struct ipath_devdata *dd = (struct ipath_devdata *) opaque;
        u64 status;
        unsigned long flags;

        if (test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
                return;

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);

        status = dd->ipath_sdma_status & IPATH_SDMA_ABORT_MASK;

        /* nothing to do */
        if (status == IPATH_SDMA_ABORT_NONE)
                goto unlock;

        /* ipath_sdma_abort() is done, waiting for interrupt */
        if (status == IPATH_SDMA_ABORT_DISARMED) {
                if (jiffies < dd->ipath_sdma_abort_intr_timeout)
                        goto resched_noprint;
                /* give up, intr got lost somewhere */
                ipath_dbg("give up waiting for SDMADISABLED intr\n");
                __set_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
                status = IPATH_SDMA_ABORT_ABORTED;
        }

        /* everything is stopped, time to clean up and restart */
        if (status == IPATH_SDMA_ABORT_ABORTED) {
                struct ipath_sdma_txreq *txp, *txpnext;
                u64 hwstatus;
                int notify = 0;

                hwstatus = ipath_read_kreg64(dd,
                                dd->ipath_kregs->kr_senddmastatus);

                if ((hwstatus & (IPATH_SDMA_STATUS_SCORE_BOARD_DRAIN_IN_PROG |
                                 IPATH_SDMA_STATUS_ABORT_IN_PROG             |
                                 IPATH_SDMA_STATUS_INTERNAL_SDMA_ENABLE)) ||
                    !(hwstatus & IPATH_SDMA_STATUS_SCB_EMPTY)) {
                        if (dd->ipath_sdma_reset_wait > 0) {
                                /* not done shutting down sdma */
                                --dd->ipath_sdma_reset_wait;
                                goto resched;
                        }
                        ipath_cdbg(VERBOSE, "gave up waiting for quiescent "
                                "status after SDMA reset, continuing\n");
                        dump_sdma_state(dd);
                }

                /* dequeue all "sent" requests */
                list_for_each_entry_safe(txp, txpnext,
                                         &dd->ipath_sdma_activelist, list) {
                        txp->callback_status = IPATH_SDMA_TXREQ_S_ABORTED;
                        if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
                                vl15_watchdog_deq(dd);
                        list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
                        notify = 1;
                }
                if (notify)
                        tasklet_hi_schedule(&dd->ipath_sdma_notify_task);

                /* reset our notion of head and tail */
                dd->ipath_sdma_descq_tail = 0;
                dd->ipath_sdma_descq_head = 0;
                dd->ipath_sdma_head_dma[0] = 0;
                dd->ipath_sdma_generation = 0;
                dd->ipath_sdma_descq_removed = dd->ipath_sdma_descq_added;

                /* Reset SendDmaLenGen */
                ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen,
                        (u64) dd->ipath_sdma_descq_cnt | (1ULL << 18));

                /* done with sdma state for a bit */
                spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);

                /*
                 * Don't restart sdma here (with the exception
                 * below). Wait until link is up to ACTIVE.  VL15 MADs
                 * used to bring the link up use PIO, and multiple link
                 * transitions otherwise cause the sdma engine to be
                 * stopped and started multiple times.
                 * The disable is done here, including the shadow,
                 * so the state is kept consistent.
                 * See ipath_restart_sdma() for the actual starting
                 * of sdma.
                 */
                spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
                dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
                ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
                                 dd->ipath_sendctrl);
                ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
                spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);

                /* make sure I see next message */
                dd->ipath_sdma_abort_jiffies = 0;

                /*
                 * Not everything that takes SDMA offline is a link
                 * status change.  If the link was up, restart SDMA.
                 */
                if (dd->ipath_flags & IPATH_LINKACTIVE)
                        ipath_restart_sdma(dd);

                goto done;
        }

resched:
        /*
         * for now, keep spinning
         * JAG - this is bad to just have default be a loop without
         * state change
         */
        if (jiffies > dd->ipath_sdma_abort_jiffies) {
                ipath_dbg("looping with status 0x%08lx\n",
                          dd->ipath_sdma_status);
                dd->ipath_sdma_abort_jiffies = jiffies + 5 * HZ;
        }
resched_noprint:
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
        if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
                tasklet_hi_schedule(&dd->ipath_sdma_abort_task);
        return;

unlock:
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
done:
        return;
}

/*
 * This is called from interrupt context.
 */
void ipath_sdma_intr(struct ipath_devdata *dd)
{
        unsigned long flags;

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);

        (void) ipath_sdma_make_progress(dd);

        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
}

static int alloc_sdma(struct ipath_devdata *dd)
{
        int ret = 0;

        /* Allocate memory for SendDMA descriptor FIFO */
        dd->ipath_sdma_descq = dma_alloc_coherent(&dd->pcidev->dev,
                SDMA_DESCQ_SZ, &dd->ipath_sdma_descq_phys, GFP_KERNEL);

        if (!dd->ipath_sdma_descq) {
                ipath_dev_err(dd, "failed to allocate SendDMA descriptor "
                        "FIFO memory\n");
                ret = -ENOMEM;
                goto done;
        }

        dd->ipath_sdma_descq_cnt =
                SDMA_DESCQ_SZ / sizeof(struct ipath_sdma_desc);

        /* Allocate memory for DMA of head register to memory */
        dd->ipath_sdma_head_dma = dma_alloc_coherent(&dd->pcidev->dev,
                PAGE_SIZE, &dd->ipath_sdma_head_phys, GFP_KERNEL);
        if (!dd->ipath_sdma_head_dma) {
                ipath_dev_err(dd, "failed to allocate SendDMA head memory\n");
                ret = -ENOMEM;
                goto cleanup_descq;
        }
        dd->ipath_sdma_head_dma[0] = 0;

        init_timer(&dd->ipath_sdma_vl15_timer);
        dd->ipath_sdma_vl15_timer.function = vl15_watchdog_timeout;
        dd->ipath_sdma_vl15_timer.data = (unsigned long)dd;
        atomic_set(&dd->ipath_sdma_vl15_count, 0);

        goto done;

cleanup_descq:
        dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
                (void *)dd->ipath_sdma_descq, dd->ipath_sdma_descq_phys);
        dd->ipath_sdma_descq = NULL;
        dd->ipath_sdma_descq_phys = 0;
done:
        return ret;
}

int setup_sdma(struct ipath_devdata *dd)
{
        int ret = 0;
        unsigned i, n;
        u64 tmp64;
        u64 senddmabufmask[3] = { 0 };
        unsigned long flags;

        ret = alloc_sdma(dd);
        if (ret)
                goto done;

        if (!dd->ipath_sdma_descq) {
                ipath_dev_err(dd, "SendDMA memory not allocated\n");
                goto done;
        }

        /*
         * Set initial status as if we had been up, then gone down.
         * This lets initial start on transition to ACTIVE be the
         * same as restart after link flap.
         */
        dd->ipath_sdma_status = IPATH_SDMA_ABORT_ABORTED;
        dd->ipath_sdma_abort_jiffies = 0;
        dd->ipath_sdma_generation = 0;
        dd->ipath_sdma_descq_tail = 0;
        dd->ipath_sdma_descq_head = 0;
        dd->ipath_sdma_descq_removed = 0;
        dd->ipath_sdma_descq_added = 0;

        /* Set SendDmaBase */
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase,
                         dd->ipath_sdma_descq_phys);
        /* Set SendDmaLenGen */
        tmp64 = dd->ipath_sdma_descq_cnt;
        tmp64 |= 1<<18; /* enable generation checking */
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, tmp64);
        /* Set SendDmaTail */
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail,
                         dd->ipath_sdma_descq_tail);
        /* Set SendDmaHeadAddr */
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr,
                         dd->ipath_sdma_head_phys);

        /*
         * Reserve all the former "kernel" piobufs, using high number range
         * so we get as many 4K buffers as possible
         */
        n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k;
        i = dd->ipath_lastport_piobuf + dd->ipath_pioreserved;
        ipath_chg_pioavailkernel(dd, i, n - i , 0);
        for (; i < n; ++i) {
                unsigned word = i / 64;
                unsigned bit = i & 63;
                BUG_ON(word >= 3);
                senddmabufmask[word] |= 1ULL << bit;
        }
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0,
                         senddmabufmask[0]);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1,
                         senddmabufmask[1]);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2,
                         senddmabufmask[2]);

        INIT_LIST_HEAD(&dd->ipath_sdma_activelist);
        INIT_LIST_HEAD(&dd->ipath_sdma_notifylist);

        tasklet_init(&dd->ipath_sdma_notify_task, sdma_notify_task,
                     (unsigned long) dd);
        tasklet_init(&dd->ipath_sdma_abort_task, sdma_abort_task,
                     (unsigned long) dd);

        /*
         * No use to turn on SDMA here, as link is probably not ACTIVE
         * Just mark it RUNNING and enable the interrupt, and let the
         * ipath_restart_sdma() on link transition to ACTIVE actually
         * enable it.
         */
        spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
        dd->ipath_sendctrl |= INFINIPATH_S_SDMAINTENABLE;
        ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
        ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
        __set_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
        spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);

done:
        return ret;
}

void teardown_sdma(struct ipath_devdata *dd)
{
        struct ipath_sdma_txreq *txp, *txpnext;
        unsigned long flags;
        dma_addr_t sdma_head_phys = 0;
        dma_addr_t sdma_descq_phys = 0;
        void *sdma_descq = NULL;
        void *sdma_head_dma = NULL;

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
        __clear_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status);
        __set_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
        __set_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status);
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);

        tasklet_kill(&dd->ipath_sdma_abort_task);
        tasklet_kill(&dd->ipath_sdma_notify_task);

        /* turn off sdma */
        spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
        dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
        ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
                dd->ipath_sendctrl);
        ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
        spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
        /* dequeue all "sent" requests */
        list_for_each_entry_safe(txp, txpnext, &dd->ipath_sdma_activelist,
                                 list) {
                txp->callback_status = IPATH_SDMA_TXREQ_S_SHUTDOWN;
                if (txp->flags & IPATH_SDMA_TXREQ_F_VL15)
                        vl15_watchdog_deq(dd);
                list_move_tail(&txp->list, &dd->ipath_sdma_notifylist);
        }
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);

        sdma_notify_taskbody(dd);

        del_timer_sync(&dd->ipath_sdma_vl15_timer);

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);

        dd->ipath_sdma_abort_jiffies = 0;

        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1, 0);
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2, 0);

        if (dd->ipath_sdma_head_dma) {
                sdma_head_dma = (void *) dd->ipath_sdma_head_dma;
                sdma_head_phys = dd->ipath_sdma_head_phys;
                dd->ipath_sdma_head_dma = NULL;
                dd->ipath_sdma_head_phys = 0;
        }

        if (dd->ipath_sdma_descq) {
                sdma_descq = dd->ipath_sdma_descq;
                sdma_descq_phys = dd->ipath_sdma_descq_phys;
                dd->ipath_sdma_descq = NULL;
                dd->ipath_sdma_descq_phys = 0;
        }

        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);

        if (sdma_head_dma)
                dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
                                  sdma_head_dma, sdma_head_phys);

        if (sdma_descq)
                dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ,
                                  sdma_descq, sdma_descq_phys);
}

/*
 * [Re]start SDMA, if we use it, and it's not already OK.
 * This is called on transition to link ACTIVE, either the first or
 * subsequent times.
 */
void ipath_restart_sdma(struct ipath_devdata *dd)
{
        unsigned long flags;
        int needed = 1;

        if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA))
                goto bail;

        /*
         * First, make sure we should, which is to say,
         * check that we are "RUNNING" (not in teardown)
         * and not "SHUTDOWN"
         */
        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);
        if (!test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status)
                || test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status))
                        needed = 0;
        else {
                __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status);
                __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status);
                __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status);
        }
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
        if (!needed) {
                ipath_dbg("invalid attempt to restart SDMA, status 0x%08lx\n",
                        dd->ipath_sdma_status);
                goto bail;
        }
        spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
        /*
         * First clear, just to be safe. Enable is only done
         * in chip on 0->1 transition
         */
        dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE;
        ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
        ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
        dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE;
        ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl);
        ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
        spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);

        /* notify upper layers */
        ipath_ib_piobufavail(dd->verbs_dev);

bail:
        return;
}

static inline void make_sdma_desc(struct ipath_devdata *dd,
        u64 *sdmadesc, u64 addr, u64 dwlen, u64 dwoffset)
{
        WARN_ON(addr & 3);
        /* SDmaPhyAddr[47:32] */
        sdmadesc[1] = addr >> 32;
        /* SDmaPhyAddr[31:0] */
        sdmadesc[0] = (addr & 0xfffffffcULL) << 32;
        /* SDmaGeneration[1:0] */
        sdmadesc[0] |= (dd->ipath_sdma_generation & 3ULL) << 30;
        /* SDmaDwordCount[10:0] */
        sdmadesc[0] |= (dwlen & 0x7ffULL) << 16;
        /* SDmaBufOffset[12:2] */
        sdmadesc[0] |= dwoffset & 0x7ffULL;
}

/*
 * This function queues one IB packet onto the send DMA queue per call.
 * The caller is responsible for checking:
 * 1) The number of send DMA descriptor entries is less than the size of
 *    the descriptor queue.
 * 2) The IB SGE addresses and lengths are 32-bit aligned
 *    (except possibly the last SGE's length)
 * 3) The SGE addresses are suitable for passing to dma_map_single().
 */
int ipath_sdma_verbs_send(struct ipath_devdata *dd,
        struct ipath_sge_state *ss, u32 dwords,
        struct ipath_verbs_txreq *tx)
{

        unsigned long flags;
        struct ipath_sge *sge;
        int ret = 0;
        u16 tail;
        __le64 *descqp;
        u64 sdmadesc[2];
        u32 dwoffset;
        dma_addr_t addr;

        if ((tx->map_len + (dwords<<2)) > dd->ipath_ibmaxlen) {
                ipath_dbg("packet size %X > ibmax %X, fail\n",
                        tx->map_len + (dwords<<2), dd->ipath_ibmaxlen);
                ret = -EMSGSIZE;
                goto fail;
        }

        spin_lock_irqsave(&dd->ipath_sdma_lock, flags);

retry:
        if (unlikely(test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status))) {
                ret = -EBUSY;
                goto unlock;
        }

        if (tx->txreq.sg_count > ipath_sdma_descq_freecnt(dd)) {
                if (ipath_sdma_make_progress(dd))
                        goto retry;
                ret = -ENOBUFS;
                goto unlock;
        }

        addr = dma_map_single(&dd->pcidev->dev, tx->txreq.map_addr,
                              tx->map_len, DMA_TO_DEVICE);
        if (dma_mapping_error(&dd->pcidev->dev, addr))
                goto ioerr;

        dwoffset = tx->map_len >> 2;
        make_sdma_desc(dd, sdmadesc, (u64) addr, dwoffset, 0);

        /* SDmaFirstDesc */
        sdmadesc[0] |= 1ULL << 12;
        if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
                sdmadesc[0] |= 1ULL << 14;      /* SDmaUseLargeBuf */

        /* write to the descq */
        tail = dd->ipath_sdma_descq_tail;
        descqp = &dd->ipath_sdma_descq[tail].qw[0];
        *descqp++ = cpu_to_le64(sdmadesc[0]);
        *descqp++ = cpu_to_le64(sdmadesc[1]);

        if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEDESC)
                tx->txreq.start_idx = tail;

        /* increment the tail */
        if (++tail == dd->ipath_sdma_descq_cnt) {
                tail = 0;
                descqp = &dd->ipath_sdma_descq[0].qw[0];
                ++dd->ipath_sdma_generation;
        }

        sge = &ss->sge;
        while (dwords) {
                u32 dw;
                u32 len;

                len = dwords << 2;
                if (len > sge->length)
                        len = sge->length;
                if (len > sge->sge_length)
                        len = sge->sge_length;
                BUG_ON(len == 0);
                dw = (len + 3) >> 2;
                addr = dma_map_single(&dd->pcidev->dev, sge->vaddr, dw << 2,
                                      DMA_TO_DEVICE);
                if (dma_mapping_error(&dd->pcidev->dev, addr))
                        goto unmap;
                make_sdma_desc(dd, sdmadesc, (u64) addr, dw, dwoffset);
                /* SDmaUseLargeBuf has to be set in every descriptor */
                if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF)
                        sdmadesc[0] |= 1ULL << 14;
                /* write to the descq */
                *descqp++ = cpu_to_le64(sdmadesc[0]);
                *descqp++ = cpu_to_le64(sdmadesc[1]);

                /* increment the tail */
                if (++tail == dd->ipath_sdma_descq_cnt) {
                        tail = 0;
                        descqp = &dd->ipath_sdma_descq[0].qw[0];
                        ++dd->ipath_sdma_generation;
                }
                sge->vaddr += len;
                sge->length -= len;
                sge->sge_length -= len;
                if (sge->sge_length == 0) {
                        if (--ss->num_sge)
                                *sge = *ss->sg_list++;
                } else if (sge->length == 0 && sge->mr != NULL) {
                        if (++sge->n >= IPATH_SEGSZ) {
                                if (++sge->m >= sge->mr->mapsz)
                                        break;
                                sge->n = 0;
                        }
                        sge->vaddr =
                                sge->mr->map[sge->m]->segs[sge->n].vaddr;
                        sge->length =
                                sge->mr->map[sge->m]->segs[sge->n].length;
                }

                dwoffset += dw;
                dwords -= dw;
        }

        if (!tail)
                descqp = &dd->ipath_sdma_descq[dd->ipath_sdma_descq_cnt].qw[0];
        descqp -= 2;
        /* SDmaLastDesc */
        descqp[0] |= cpu_to_le64(1ULL << 11);
        if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_INTREQ) {
                /* SDmaIntReq */
                descqp[0] |= cpu_to_le64(1ULL << 15);
        }

        /* Commit writes to memory and advance the tail on the chip */
        wmb();
        ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, tail);

        tx->txreq.next_descq_idx = tail;
        tx->txreq.callback_status = IPATH_SDMA_TXREQ_S_OK;
        dd->ipath_sdma_descq_tail = tail;
        dd->ipath_sdma_descq_added += tx->txreq.sg_count;
        list_add_tail(&tx->txreq.list, &dd->ipath_sdma_activelist);
        if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_VL15)
                vl15_watchdog_enq(dd);
        goto unlock;

unmap:
        while (tail != dd->ipath_sdma_descq_tail) {
                if (!tail)
                        tail = dd->ipath_sdma_descq_cnt - 1;
                else
                        tail--;
                unmap_desc(dd, tail);
        }
ioerr:
        ret = -EIO;
unlock:
        spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags);
fail:
        return ret;
}