Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / infiniband / core / cq.c
blobbc79ca8215d7c25ff22c2a901db04b209b49052a
1 /*
2 * Copyright (c) 2015 HGST, a Western Digital Company.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/slab.h>
16 #include <rdma/ib_verbs.h>
18 /* # of WCs to poll for with a single call to ib_poll_cq */
19 #define IB_POLL_BATCH 16
21 /* # of WCs to iterate over before yielding */
22 #define IB_POLL_BUDGET_IRQ 256
23 #define IB_POLL_BUDGET_WORKQUEUE 65536
25 #define IB_POLL_FLAGS \
26 (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
28 static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *poll_wc)
30 int i, n, completed = 0;
31 struct ib_wc *wcs = poll_wc ? : cq->wc;
34 * budget might be (-1) if the caller does not
35 * want to bound this call, thus we need unsigned
36 * minimum here.
38 while ((n = ib_poll_cq(cq, min_t(u32, IB_POLL_BATCH,
39 budget - completed), wcs)) > 0) {
40 for (i = 0; i < n; i++) {
41 struct ib_wc *wc = &wcs[i];
43 if (wc->wr_cqe)
44 wc->wr_cqe->done(cq, wc);
45 else
46 WARN_ON_ONCE(wc->status == IB_WC_SUCCESS);
49 completed += n;
51 if (n != IB_POLL_BATCH ||
52 (budget != -1 && completed >= budget))
53 break;
56 return completed;
59 /**
60 * ib_process_direct_cq - process a CQ in caller context
61 * @cq: CQ to process
62 * @budget: number of CQEs to poll for
64 * This function is used to process all outstanding CQ entries.
65 * It does not offload CQ processing to a different context and does
66 * not ask for completion interrupts from the HCA.
67 * Using direct processing on CQ with non IB_POLL_DIRECT type may trigger
68 * concurrent processing.
70 * Note: do not pass -1 as %budget unless it is guaranteed that the number
71 * of completions that will be processed is small.
73 int ib_process_cq_direct(struct ib_cq *cq, int budget)
75 struct ib_wc wcs[IB_POLL_BATCH];
77 return __ib_process_cq(cq, budget, wcs);
79 EXPORT_SYMBOL(ib_process_cq_direct);
81 static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
83 WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);
86 static int ib_poll_handler(struct irq_poll *iop, int budget)
88 struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
89 int completed;
91 completed = __ib_process_cq(cq, budget, NULL);
92 if (completed < budget) {
93 irq_poll_complete(&cq->iop);
94 if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
95 irq_poll_sched(&cq->iop);
98 return completed;
101 static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
103 irq_poll_sched(&cq->iop);
106 static void ib_cq_poll_work(struct work_struct *work)
108 struct ib_cq *cq = container_of(work, struct ib_cq, work);
109 int completed;
111 completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, NULL);
112 if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
113 ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
114 queue_work(ib_comp_wq, &cq->work);
117 static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
119 queue_work(ib_comp_wq, &cq->work);
123 * __ib_alloc_cq - allocate a completion queue
124 * @dev: device to allocate the CQ for
125 * @private: driver private data, accessible from cq->cq_context
126 * @nr_cqe: number of CQEs to allocate
127 * @comp_vector: HCA completion vectors for this CQ
128 * @poll_ctx: context to poll the CQ from.
129 * @caller: module owner name.
131 * This is the proper interface to allocate a CQ for in-kernel users. A
132 * CQ allocated with this interface will automatically be polled from the
133 * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
134 * to use this CQ abstraction.
136 struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
137 int nr_cqe, int comp_vector,
138 enum ib_poll_context poll_ctx, const char *caller)
140 struct ib_cq_init_attr cq_attr = {
141 .cqe = nr_cqe,
142 .comp_vector = comp_vector,
144 struct ib_cq *cq;
145 int ret = -ENOMEM;
147 cq = dev->create_cq(dev, &cq_attr, NULL, NULL);
148 if (IS_ERR(cq))
149 return cq;
151 cq->device = dev;
152 cq->uobject = NULL;
153 cq->event_handler = NULL;
154 cq->cq_context = private;
155 cq->poll_ctx = poll_ctx;
156 atomic_set(&cq->usecnt, 0);
158 cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
159 if (!cq->wc)
160 goto out_destroy_cq;
162 cq->res.type = RDMA_RESTRACK_CQ;
163 cq->res.kern_name = caller;
164 rdma_restrack_add(&cq->res);
166 switch (cq->poll_ctx) {
167 case IB_POLL_DIRECT:
168 cq->comp_handler = ib_cq_completion_direct;
169 break;
170 case IB_POLL_SOFTIRQ:
171 cq->comp_handler = ib_cq_completion_softirq;
173 irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
174 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
175 break;
176 case IB_POLL_WORKQUEUE:
177 cq->comp_handler = ib_cq_completion_workqueue;
178 INIT_WORK(&cq->work, ib_cq_poll_work);
179 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
180 break;
181 default:
182 ret = -EINVAL;
183 goto out_free_wc;
186 return cq;
188 out_free_wc:
189 kfree(cq->wc);
190 rdma_restrack_del(&cq->res);
191 out_destroy_cq:
192 cq->device->destroy_cq(cq);
193 return ERR_PTR(ret);
195 EXPORT_SYMBOL(__ib_alloc_cq);
198 * ib_free_cq - free a completion queue
199 * @cq: completion queue to free.
201 void ib_free_cq(struct ib_cq *cq)
203 int ret;
205 if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
206 return;
208 switch (cq->poll_ctx) {
209 case IB_POLL_DIRECT:
210 break;
211 case IB_POLL_SOFTIRQ:
212 irq_poll_disable(&cq->iop);
213 break;
214 case IB_POLL_WORKQUEUE:
215 cancel_work_sync(&cq->work);
216 break;
217 default:
218 WARN_ON_ONCE(1);
221 kfree(cq->wc);
222 rdma_restrack_del(&cq->res);
223 ret = cq->device->destroy_cq(cq);
224 WARN_ON_ONCE(ret);
226 EXPORT_SYMBOL(ib_free_cq);