2 * Copyright (c) 2015 HGST, a Western Digital Company.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/slab.h>
16 #include <rdma/ib_verbs.h>
18 /* # of WCs to poll for with a single call to ib_poll_cq */
19 #define IB_POLL_BATCH 16
21 /* # of WCs to iterate over before yielding */
22 #define IB_POLL_BUDGET_IRQ 256
23 #define IB_POLL_BUDGET_WORKQUEUE 65536
25 #define IB_POLL_FLAGS \
26 (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
28 static int __ib_process_cq(struct ib_cq
*cq
, int budget
, struct ib_wc
*poll_wc
)
30 int i
, n
, completed
= 0;
31 struct ib_wc
*wcs
= poll_wc
? : cq
->wc
;
34 * budget might be (-1) if the caller does not
35 * want to bound this call, thus we need unsigned
38 while ((n
= ib_poll_cq(cq
, min_t(u32
, IB_POLL_BATCH
,
39 budget
- completed
), wcs
)) > 0) {
40 for (i
= 0; i
< n
; i
++) {
41 struct ib_wc
*wc
= &wcs
[i
];
44 wc
->wr_cqe
->done(cq
, wc
);
46 WARN_ON_ONCE(wc
->status
== IB_WC_SUCCESS
);
51 if (n
!= IB_POLL_BATCH
||
52 (budget
!= -1 && completed
>= budget
))
60 * ib_process_direct_cq - process a CQ in caller context
62 * @budget: number of CQEs to poll for
64 * This function is used to process all outstanding CQ entries.
65 * It does not offload CQ processing to a different context and does
66 * not ask for completion interrupts from the HCA.
67 * Using direct processing on CQ with non IB_POLL_DIRECT type may trigger
68 * concurrent processing.
70 * Note: do not pass -1 as %budget unless it is guaranteed that the number
71 * of completions that will be processed is small.
73 int ib_process_cq_direct(struct ib_cq
*cq
, int budget
)
75 struct ib_wc wcs
[IB_POLL_BATCH
];
77 return __ib_process_cq(cq
, budget
, wcs
);
79 EXPORT_SYMBOL(ib_process_cq_direct
);
81 static void ib_cq_completion_direct(struct ib_cq
*cq
, void *private)
83 WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq
);
86 static int ib_poll_handler(struct irq_poll
*iop
, int budget
)
88 struct ib_cq
*cq
= container_of(iop
, struct ib_cq
, iop
);
91 completed
= __ib_process_cq(cq
, budget
, NULL
);
92 if (completed
< budget
) {
93 irq_poll_complete(&cq
->iop
);
94 if (ib_req_notify_cq(cq
, IB_POLL_FLAGS
) > 0)
95 irq_poll_sched(&cq
->iop
);
101 static void ib_cq_completion_softirq(struct ib_cq
*cq
, void *private)
103 irq_poll_sched(&cq
->iop
);
106 static void ib_cq_poll_work(struct work_struct
*work
)
108 struct ib_cq
*cq
= container_of(work
, struct ib_cq
, work
);
111 completed
= __ib_process_cq(cq
, IB_POLL_BUDGET_WORKQUEUE
, NULL
);
112 if (completed
>= IB_POLL_BUDGET_WORKQUEUE
||
113 ib_req_notify_cq(cq
, IB_POLL_FLAGS
) > 0)
114 queue_work(ib_comp_wq
, &cq
->work
);
117 static void ib_cq_completion_workqueue(struct ib_cq
*cq
, void *private)
119 queue_work(ib_comp_wq
, &cq
->work
);
123 * __ib_alloc_cq - allocate a completion queue
124 * @dev: device to allocate the CQ for
125 * @private: driver private data, accessible from cq->cq_context
126 * @nr_cqe: number of CQEs to allocate
127 * @comp_vector: HCA completion vectors for this CQ
128 * @poll_ctx: context to poll the CQ from.
129 * @caller: module owner name.
131 * This is the proper interface to allocate a CQ for in-kernel users. A
132 * CQ allocated with this interface will automatically be polled from the
133 * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
134 * to use this CQ abstraction.
136 struct ib_cq
*__ib_alloc_cq(struct ib_device
*dev
, void *private,
137 int nr_cqe
, int comp_vector
,
138 enum ib_poll_context poll_ctx
, const char *caller
)
140 struct ib_cq_init_attr cq_attr
= {
142 .comp_vector
= comp_vector
,
147 cq
= dev
->create_cq(dev
, &cq_attr
, NULL
, NULL
);
153 cq
->event_handler
= NULL
;
154 cq
->cq_context
= private;
155 cq
->poll_ctx
= poll_ctx
;
156 atomic_set(&cq
->usecnt
, 0);
158 cq
->wc
= kmalloc_array(IB_POLL_BATCH
, sizeof(*cq
->wc
), GFP_KERNEL
);
162 cq
->res
.type
= RDMA_RESTRACK_CQ
;
163 cq
->res
.kern_name
= caller
;
164 rdma_restrack_add(&cq
->res
);
166 switch (cq
->poll_ctx
) {
168 cq
->comp_handler
= ib_cq_completion_direct
;
170 case IB_POLL_SOFTIRQ
:
171 cq
->comp_handler
= ib_cq_completion_softirq
;
173 irq_poll_init(&cq
->iop
, IB_POLL_BUDGET_IRQ
, ib_poll_handler
);
174 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
176 case IB_POLL_WORKQUEUE
:
177 cq
->comp_handler
= ib_cq_completion_workqueue
;
178 INIT_WORK(&cq
->work
, ib_cq_poll_work
);
179 ib_req_notify_cq(cq
, IB_CQ_NEXT_COMP
);
190 rdma_restrack_del(&cq
->res
);
192 cq
->device
->destroy_cq(cq
);
195 EXPORT_SYMBOL(__ib_alloc_cq
);
198 * ib_free_cq - free a completion queue
199 * @cq: completion queue to free.
201 void ib_free_cq(struct ib_cq
*cq
)
205 if (WARN_ON_ONCE(atomic_read(&cq
->usecnt
)))
208 switch (cq
->poll_ctx
) {
211 case IB_POLL_SOFTIRQ
:
212 irq_poll_disable(&cq
->iop
);
214 case IB_POLL_WORKQUEUE
:
215 cancel_work_sync(&cq
->work
);
222 rdma_restrack_del(&cq
->res
);
223 ret
= cq
->device
->destroy_cq(cq
);
226 EXPORT_SYMBOL(ib_free_cq
);