?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'origin/master'
[unleashed/lotheac.git] / usr / src / uts / common / io / ib / clients / daplt / daplt.c
blob4a766d728223ac8c7c9c942257c62940fd44e65d
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 *
26 * UDAPL kernel agent
27 */
28
29 #include <sys/types.h>
30 #include <sys/errno.h>
31 #include <sys/debug.h>
32 #include <sys/stropts.h>
33 #include <sys/stream.h>
34 #include <sys/strlog.h>
35 #include <sys/cmn_err.h>
36 #include <sys/kmem.h>
37 #include <sys/conf.h>
38 #include <sys/stat.h>
39 #include <sys/modctl.h>
40 #include <sys/kstat.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 #include <sys/strsun.h>
44 #include <sys/taskq.h>
45 #include <sys/open.h>
46 #include <sys/uio.h>
47 #include <sys/cpuvar.h>
48 #include <sys/atomic.h>
49 #include <sys/sysmacros.h>
50 #include <sys/esunddi.h>
51 #include <sys/avl.h>
52 #include <sys/cred.h>
53 #include <sys/note.h>
54 #include <sys/ib/ibtl/ibti.h>
55 #include <sys/socket.h>
56 #include <netinet/in.h>
57 #include <daplt_if.h>
58 #include <daplt.h>
59
60 /*
61 * The following variables support the debug log buffer scheme.
62 */
63 #ifdef DEBUG
64 static char daplka_dbgbuf[0x80000];
65 #else /* DEBUG */
66 static char daplka_dbgbuf[0x4000];
67 #endif /* DEBUG */
68 static int daplka_dbgsize = sizeof (daplka_dbgbuf);
69 static size_t daplka_dbgnext;
70 static int daplka_dbginit = 0;
71 static kmutex_t daplka_dbglock;
72 _NOTE(MUTEX_PROTECTS_DATA(daplka_dbglock,
73 daplka_dbgbuf
74 daplka_dbgnext))
75
76 static int daplka_dbg = 0x0103;
77 static void daplka_console(const char *, ...);
78 static void daplka_debug(const char *, ...);
79 static int daplka_apm = 0x1; /* default enable */
80 static int daplka_failback = 0x1; /* default enable */
81 static int daplka_query_aft_setaltpath = 10;
82
83 #define DERR \
84 if (daplka_dbg & 0x100) \
85 daplka_debug
86
87 #ifdef DEBUG
88
89 #define DINFO \
90 daplka_console
91
92 #define D1 \
93 if (daplka_dbg & 0x01) \
94 daplka_debug
95 #define D2 \
96 if (daplka_dbg & 0x02) \
97 daplka_debug
98 #define D3 \
99 if (daplka_dbg & 0x04) \
100 daplka_debug
101 #define D4 \
102 if (daplka_dbg & 0x08) \
103 daplka_debug
104
105 #else /* DEBUG */
106
107 #define DINFO if (0) printf
108 #define D1 if (0) printf
109 #define D2 if (0) printf
110 #define D3 if (0) printf
111 #define D4 if (0) printf
112
113 #endif /* DEBUG */
114
115 /*
116 * driver entry points
117 */
118 static int daplka_open(dev_t *, int, int, struct cred *);
119 static int daplka_close(dev_t, int, int, struct cred *);
120 static int daplka_attach(dev_info_t *, ddi_attach_cmd_t);
121 static int daplka_detach(dev_info_t *, ddi_detach_cmd_t);
122 static int daplka_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
123 static int daplka_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
124
125 /*
126 * types of ioctls
127 */
128 static int daplka_common_ioctl(int, minor_t, intptr_t, int, cred_t *, int *);
129 static int daplka_misc_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
130 cred_t *, int *);
131 static int daplka_ep_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
132 cred_t *, int *);
133 static int daplka_evd_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
134 cred_t *, int *);
135 static int daplka_mr_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
136 cred_t *, int *);
137 static int daplka_cno_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
138 cred_t *, int *);
139 static int daplka_pd_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
140 cred_t *, int *);
141 static int daplka_sp_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
142 cred_t *, int *);
143 static int daplka_srq_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
144 cred_t *, int *);
145
146 /*
147 * common ioctls and supporting functions
148 */
149 static int daplka_ia_create(minor_t, intptr_t, int, cred_t *, int *);
150 static int daplka_ia_destroy(daplka_resource_t *);
151
152 /*
153 * EP ioctls and supporting functions
154 */
155 static int daplka_ep_create(daplka_ia_resource_t *, intptr_t, int,
156 cred_t *, int *);
157 static int daplka_ep_modify(daplka_ia_resource_t *, intptr_t, int,
158 cred_t *, int *);
159 static int daplka_ep_free(daplka_ia_resource_t *, intptr_t, int,
160 cred_t *, int *);
161 static int daplka_ep_connect(daplka_ia_resource_t *, intptr_t, int,
162 cred_t *, int *);
163 static int daplka_ep_disconnect(daplka_ia_resource_t *, intptr_t, int,
164 cred_t *, int *);
165 static int daplka_ep_reinit(daplka_ia_resource_t *, intptr_t, int,
166 cred_t *, int *);
167 static int daplka_ep_destroy(daplka_resource_t *);
168 static void daplka_hash_ep_free(void *);
169 static int daplka_ep_failback(void *objp, void *arg);
170 static int daplka_ep_altpath(daplka_ep_resource_t *, ib_gid_t *);
171
172 static uint32_t daplka_ep_get_state(daplka_ep_resource_t *);
173 static void daplka_ep_set_state(daplka_ep_resource_t *, uint32_t, uint32_t);
174 static boolean_t daplka_ep_transition_is_valid(uint32_t, uint32_t);
175 static daplka_timer_info_t *daplka_timer_info_alloc(daplka_ep_resource_t *);
176 static void daplka_timer_info_free(daplka_timer_info_t *);
177 static void daplka_timer_handler(void *);
178 static void daplka_timer_dispatch(void *);
179 static void daplka_timer_thread(void *);
180 static int daplka_cancel_timer(daplka_ep_resource_t *);
181 static void daplka_hash_timer_free(void *);
182
183 /*
184 * EVD ioctls and supporting functions
185 */
186 static int daplka_evd_create(daplka_ia_resource_t *, intptr_t, int,
187 cred_t *, int *);
188 static int daplka_cq_resize(daplka_ia_resource_t *, intptr_t, int,
189 cred_t *, int *);
190 static int daplka_evd_free(daplka_ia_resource_t *, intptr_t, int,
191 cred_t *, int *);
192 static int daplka_event_poll(daplka_ia_resource_t *, intptr_t, int,
193 cred_t *, int *);
194 static int daplka_evd_destroy(daplka_resource_t *);
195 static void daplka_cq_handler(ibt_cq_hdl_t, void *);
196 static void daplka_evd_wakeup(daplka_evd_resource_t *,
197 daplka_evd_event_list_t *, daplka_evd_event_t *);
198 static void daplka_evd_event_enqueue(daplka_evd_event_list_t *,
199 daplka_evd_event_t *);
200 static daplka_evd_event_t *daplka_evd_event_dequeue(daplka_evd_event_list_t *);
201 static void daplka_hash_evd_free(void *);
202
203
204 /*
205 * SRQ ioctls and supporting functions
206 */
207 static int daplka_srq_create(daplka_ia_resource_t *, intptr_t, int,
208 cred_t *, int *);
209 static int daplka_srq_resize(daplka_ia_resource_t *, intptr_t, int,
210 cred_t *, int *);
211 static int daplka_srq_free(daplka_ia_resource_t *, intptr_t, int,
212 cred_t *, int *);
213 static int daplka_srq_destroy(daplka_resource_t *);
214 static void daplka_hash_srq_free(void *);
215
216 /*
217 * Miscellaneous ioctls
218 */
219 static int daplka_cr_accept(daplka_ia_resource_t *, intptr_t, int,
220 cred_t *, int *);
221 static int daplka_cr_reject(daplka_ia_resource_t *, intptr_t, int,
222 cred_t *, int *);
223 static int daplka_cr_handoff(daplka_ia_resource_t *, intptr_t, int,
224 cred_t *, int *);
225 static int daplka_ia_query(daplka_ia_resource_t *, intptr_t, int,
226 cred_t *, int *);
227
228 /*
229 * PD ioctls and supporting functions
230 */
231 static int daplka_pd_alloc(daplka_ia_resource_t *, intptr_t, int,
232 cred_t *, int *);
233 static int daplka_pd_free(daplka_ia_resource_t *, intptr_t, int,
234 cred_t *, int *);
235 static int daplka_pd_destroy(daplka_resource_t *);
236 static void daplka_hash_pd_free(void *);
237
238 /*
239 * SP ioctls and supporting functions
240 */
241 static int daplka_service_register(daplka_ia_resource_t *, intptr_t, int,
242 cred_t *, int *);
243 static int daplka_service_deregister(daplka_ia_resource_t *, intptr_t, int,
244 cred_t *, int *);
245 static int daplka_sp_destroy(daplka_resource_t *);
246 static void daplka_hash_sp_free(void *);
247 static void daplka_hash_sp_unref(void *);
248
249 /*
250 * MR ioctls and supporting functions
251 */
252 static int daplka_mr_register(daplka_ia_resource_t *, intptr_t, int,
253 cred_t *, int *);
254 static int daplka_mr_register_lmr(daplka_ia_resource_t *, intptr_t, int,
255 cred_t *, int *);
256 static int daplka_mr_register_shared(daplka_ia_resource_t *, intptr_t, int,
257 cred_t *, int *);
258 static int daplka_mr_deregister(daplka_ia_resource_t *, intptr_t, int,
259 cred_t *, int *);
260 static int daplka_mr_sync(daplka_ia_resource_t *, intptr_t, int,
261 cred_t *, int *);
262 static int daplka_mr_destroy(daplka_resource_t *);
263 static void daplka_hash_mr_free(void *);
264 static void daplka_shared_mr_free(daplka_mr_resource_t *);
265
266 /*
267 * MW ioctls and supporting functions
268 */
269 static int daplka_mw_alloc(daplka_ia_resource_t *, intptr_t, int,
270 cred_t *, int *);
271 static int daplka_mw_free(daplka_ia_resource_t *, intptr_t, int,
272 cred_t *, int *);
273 static int daplka_mw_destroy(daplka_resource_t *);
274 static void daplka_hash_mw_free(void *);
275
276 /*
277 * CNO ioctls and supporting functions
278 */
279 static int daplka_cno_alloc(daplka_ia_resource_t *, intptr_t, int,
280 cred_t *, int *);
281 static int daplka_cno_free(daplka_ia_resource_t *, intptr_t, int,
282 cred_t *, int *);
283 static int daplka_cno_wait(daplka_ia_resource_t *, intptr_t, int,
284 cred_t *, int *);
285 static int daplka_cno_destroy(daplka_resource_t *);
286 static void daplka_hash_cno_free(void *);
287
288 /*
289 * CM handlers
290 */
291 static ibt_cm_status_t daplka_cm_rc_handler(void *, ibt_cm_event_t *,
292 ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
293
294 static ibt_cm_status_t daplka_cm_service_handler(void *, ibt_cm_event_t *,
295 ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
296
297 static ibt_cm_status_t daplka_cm_service_req(daplka_sp_resource_t *,
298 ibt_cm_event_t *, ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
299
300 /*
301 * resource management routines
302 */
303 static int daplka_resource_reserve(minor_t *);
304 static int daplka_resource_insert(minor_t, daplka_resource_t *);
305 static daplka_resource_t *daplka_resource_remove(minor_t rnum);
306 static daplka_resource_t *daplka_resource_lookup(minor_t);
307 static void daplka_resource_init(void);
308 static void daplka_resource_fini(void);
309 static struct daplka_resource_table daplka_resource;
310
311 /*
312 * hash table routines
313 */
314 static int daplka_hash_insert(daplka_hash_table_t *, uint64_t *, void *);
315 static int daplka_hash_remove(daplka_hash_table_t *, uint64_t, void **);
316 static void daplka_hash_walk(daplka_hash_table_t *, int (*)(void *, void *),
317 void *, krw_t);
318 static void *daplka_hash_lookup(daplka_hash_table_t *, uint64_t);
319 static int daplka_hash_create(daplka_hash_table_t *, uint_t,
320 void (*)(void *), void (*)(void *));
321 static void daplka_hash_destroy(daplka_hash_table_t *);
322 static uint32_t daplka_hash_getsize(daplka_hash_table_t *);
323 static void daplka_hash_generic_lookup(void *);
324
325 static uint32_t daplka_timer_hkey_gen();
326
327 /*
328 * async event handlers
329 */
330 static void daplka_async_event_create(ibt_async_code_t, ibt_async_event_t *,
331 uint64_t, daplka_ia_resource_t *);
332 static void daplka_rc_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
333 ibt_async_event_t *);
334 static void daplka_cq_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
335 ibt_async_event_t *);
336 static void daplka_un_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
337 ibt_async_event_t *);
338 static void daplka_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
339 ibt_async_event_t *);
340 static void daplka_sm_notice_handler(void *, ib_gid_t, ibt_subnet_event_code_t,
341 ibt_subnet_event_t *event);
342 static void daplka_sm_gid_avail(ib_gid_t *, ib_gid_t *);
343
344 /*
345 * IBTF wrappers and default limits used for resource accounting
346 */
347 static boolean_t daplka_accounting_enabled = B_TRUE;
348 static uint32_t daplka_max_qp_percent = 100;
349 static uint32_t daplka_max_cq_percent = 100;
350 static uint32_t daplka_max_pd_percent = 100;
351 static uint32_t daplka_max_mw_percent = 100;
352 static uint32_t daplka_max_mr_percent = 100;
353 static uint32_t daplka_max_srq_percent = 100;
354
355 static ibt_status_t
356 daplka_ibt_alloc_rc_channel(daplka_ep_resource_t *, ibt_hca_hdl_t,
357 ibt_chan_alloc_flags_t, ibt_rc_chan_alloc_args_t *,
358 ibt_channel_hdl_t *, ibt_chan_sizes_t *);
359
360 static ibt_status_t
361 daplka_ibt_free_channel(daplka_ep_resource_t *, ibt_channel_hdl_t);
362
363 static ibt_status_t
364 daplka_ibt_alloc_cq(daplka_evd_resource_t *, ibt_hca_hdl_t,
365 ibt_cq_attr_t *, ibt_cq_hdl_t *, uint_t *);
366
367 static ibt_status_t
368 daplka_ibt_free_cq(daplka_evd_resource_t *, ibt_cq_hdl_t);
369
370 static ibt_status_t
371 daplka_ibt_alloc_pd(daplka_pd_resource_t *, ibt_hca_hdl_t,
372 ibt_pd_flags_t, ibt_pd_hdl_t *);
373
374 static ibt_status_t
375 daplka_ibt_free_pd(daplka_pd_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t);
376
377 static ibt_status_t
378 daplka_ibt_alloc_mw(daplka_mw_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t,
379 ibt_mw_flags_t, ibt_mw_hdl_t *, ibt_rkey_t *);
380
381 static ibt_status_t
382 daplka_ibt_free_mw(daplka_mw_resource_t *, ibt_hca_hdl_t, ibt_mw_hdl_t);
383
384 static ibt_status_t
385 daplka_ibt_register_mr(daplka_mr_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t,
386 ibt_mr_attr_t *, ibt_mr_hdl_t *, ibt_mr_desc_t *);
387
388 static ibt_status_t
389 daplka_ibt_register_shared_mr(daplka_mr_resource_t *, ibt_hca_hdl_t,
390 ibt_mr_hdl_t, ibt_pd_hdl_t, ibt_smr_attr_t *, ibt_mr_hdl_t *,
391 ibt_mr_desc_t *);
392
393 static ibt_status_t
394 daplka_ibt_deregister_mr(daplka_mr_resource_t *, ibt_hca_hdl_t, ibt_mr_hdl_t);
395
396 static ibt_status_t
397 daplka_ibt_alloc_srq(daplka_srq_resource_t *, ibt_hca_hdl_t, ibt_srq_flags_t,
398 ibt_pd_hdl_t, ibt_srq_sizes_t *, ibt_srq_hdl_t *, ibt_srq_sizes_t *);
399
400 static ibt_status_t
401 daplka_ibt_free_srq(daplka_srq_resource_t *, ibt_srq_hdl_t);
402
403 /*
404 * macros for manipulating resource objects.
405 * these macros can be used on objects that begin with a
406 * daplka_resource_t header.
407 */
408 #define DAPLKA_RS_REFCNT(rp) ((rp)->header.rs_refcnt)
409
410 #define DAPLKA_RS_REF(rp) { \
411 mutex_enter(&(rp)->header.rs_reflock); \
412 (rp)->header.rs_refcnt++; \
413 ASSERT((rp)->header.rs_refcnt != 0); \
414 mutex_exit(&(rp)->header.rs_reflock); \
415 }
416
417 #define DAPLKA_RS_UNREF(rp) { \
418 mutex_enter(&(rp)->header.rs_reflock); \
419 ASSERT((rp)->header.rs_refcnt != 0); \
420 if (--(rp)->header.rs_refcnt == 0) { \
421 ASSERT((rp)->header.rs_free != NULL); \
422 mutex_exit(&(rp)->header.rs_reflock); \
423 (rp)->header.rs_free((daplka_resource_t *)rp); \
424 } else { \
425 mutex_exit(&(rp)->header.rs_reflock); \
426 } \
427 }
428
429 #define DAPLKA_RS_INIT(rp, type, rnum, free_func) { \
430 (rp)->header.rs_refcnt = 1; \
431 (rp)->header.rs_type = (type); \
432 (rp)->header.rs_rnum = (rnum); \
433 (rp)->header.rs_charged = 0; \
434 (rp)->header.rs_free = (free_func); \
435 mutex_init(&(rp)->header.rs_reflock, NULL, \
436 MUTEX_DRIVER, NULL); \
437 }
438
439 #define DAPLKA_RS_FINI(rp) { \
440 mutex_destroy(&(rp)->header.rs_reflock); \
441 }
442
443 #define DAPLKA_RS_ACCT_INC(rp, cnt) { \
444 atomic_add_32(&(rp)->header.rs_charged, (cnt)); \
445 }
446 #define DAPLKA_RS_ACCT_DEC(rp, cnt) { \
447 atomic_add_32(&(rp)->header.rs_charged, -(cnt)); \
448 }
449 #define DAPLKA_RS_ACCT_CHARGED(rp) ((rp)->header.rs_charged)
450
451 #define DAPLKA_RS_RNUM(rp) ((rp)->header.rs_rnum)
452 #define DAPLKA_RS_TYPE(rp) ((rp)->header.rs_type)
453 #define DAPLKA_RS_RESERVED(rp) ((intptr_t)(rp) == DAPLKA_RC_RESERVED)
454
455 /*
456 * depending on the timeout value does a cv_wait_sig or cv_timedwait_sig
457 */
458 #define DAPLKA_EVD_WAIT(cvp, mp, timeout) \
459 ((timeout) == LONG_MAX) ? cv_wait_sig((cvp), (mp)) : \
460 cv_timedwait_sig((cvp), (mp), (timeout))
461
462 #define DAPLKA_HOLD_HCA_WITHOUT_LOCK(hca) ((hca)->hca_ref_cnt++)
463 #define DAPLKA_RELE_HCA_WITHOUT_LOCK(hca) ((hca)->hca_ref_cnt--)
464
465 #define DAPLKA_HOLD_HCA(dp, hca) { \
466 mutex_enter(&(dp)->daplka_mutex); \
467 DAPLKA_HOLD_HCA_WITHOUT_LOCK(hca); \
468 mutex_exit(&(dp)->daplka_mutex); \
469 }
470
471 #define DAPLKA_RELE_HCA(dp, hca) { \
472 mutex_enter(&(dp)->daplka_mutex); \
473 DAPLKA_RELE_HCA_WITHOUT_LOCK(hca); \
474 mutex_exit(&(dp)->daplka_mutex); \
475 }
476
477 #define DAPLKA_HCA_BUSY(hca) \
478 ((hca)->hca_ref_cnt != 0 || \
479 (hca)->hca_qp_count != 0 || \
480 (hca)->hca_cq_count != 0 || \
481 (hca)->hca_pd_count != 0 || \
482 (hca)->hca_mw_count != 0 || \
483 (hca)->hca_mr_count != 0)
484
485
486 static struct cb_ops daplka_cb_ops = {
487 daplka_open, /* cb_open */
488 daplka_close, /* cb_close */
489 nodev, /* cb_strategy */
490 nodev, /* cb_print */
491 nodev, /* cb_dump */
492 nodev, /* cb_read */
493 nodev, /* cb_write */
494 daplka_ioctl, /* cb_ioctl */
495 nodev, /* cb_devmap */
496 nodev, /* cb_mmap */
497 nodev, /* cb_segmap */
498 nochpoll, /* cb_chpoll */
499 ddi_prop_op, /* cb_prop_op */
500 NULL, /* cb_stream */
501 D_NEW | D_MP, /* cb_flag */
502 CB_REV, /* rev */
503 nodev, /* int (*cb_aread)() */
504 nodev /* int (*cb_awrite)() */
505 };
506
507 static struct dev_ops daplka_ops = {
508 DEVO_REV, /* devo_rev */
509 0, /* devo_refcnt */
510 daplka_info, /* devo_getinfo */
511 nulldev, /* devo_identify */
512 nulldev, /* devo_probe */
513 daplka_attach, /* devo_attach */
514 daplka_detach, /* devo_detach */
515 nodev, /* devo_reset */
516 &daplka_cb_ops, /* devo_cb_ops */
517 NULL, /* devo_bus_ops */
518 nulldev, /* power */
519 ddi_quiesce_not_needed, /* devo_quiesce */
520 };
521
522 /*
523 * Module linkage information for the kernel.
524 */
525 static struct modldrv modldrv = {
526 &mod_driverops,
527 "uDAPL Service Driver",
528 &daplka_ops,
529 };
530
531 static struct modlinkage modlinkage = {
532 #ifdef _LP64
533 MODREV_1, { (void *) &modldrv, NULL, NULL, NULL, NULL, NULL, NULL }
534 #else
535 MODREV_1, { (void *) &modldrv, NULL, NULL, NULL }
536 #endif
537 };
538
539 /*
540 * daplka_dev holds global driver state and a list of HCAs
541 */
542 static daplka_t *daplka_dev = NULL;
543 static void *daplka_state = NULL;
544
545 /*
546 * global SP hash table
547 */
548 static daplka_hash_table_t daplka_global_sp_htbl;
549
550 /*
551 * timer_info hash table
552 */
553 static daplka_hash_table_t daplka_timer_info_htbl;
554 static uint32_t daplka_timer_hkey = 0;
555
556 /*
557 * shared MR avl tree
558 */
559 static avl_tree_t daplka_shared_mr_tree;
560 static kmutex_t daplka_shared_mr_lock;
561 static int daplka_shared_mr_cmp(const void *, const void *);
562 _NOTE(MUTEX_PROTECTS_DATA(daplka_shared_mr_lock,
563 daplka_shared_mr_tree))
564
565 /*
566 * default kmem flags used by this driver
567 */
568 static int daplka_km_flags = KM_SLEEP;
569
570 /*
571 * taskq used for handling background tasks
572 */
573 static taskq_t *daplka_taskq = NULL;
574
575 /*
576 * daplka_cm_delay is the length of time the active
577 * side needs to wait before timing out on the REP message.
578 */
579 static clock_t daplka_cm_delay = 60000000;
580
581 /*
582 * modunload will fail if pending_close is non-zero
583 */
584 static uint32_t daplka_pending_close = 0;
585
586 static struct ibt_clnt_modinfo_s daplka_clnt_modinfo = {
587 IBTI_V_CURR,
588 IBT_USER,
589 daplka_async_handler,
590 NULL,
591 DAPLKA_DRV_NAME
592 };
593
594 /*
595 * Module Installation
596 */
597 int
598 _init(void)
599 {
600 int status;
601
602 status = ddi_soft_state_init(&daplka_state, sizeof (daplka_t), 1);
603 if (status != 0) {
604 return (status);
605 }
606
607 mutex_init(&daplka_dbglock, NULL, MUTEX_DRIVER, NULL);
608 bzero(daplka_dbgbuf, sizeof (daplka_dbgbuf));
609 daplka_dbgnext = 0;
610 daplka_dbginit = 1;
611
612 daplka_resource_init();
613
614 status = mod_install(&modlinkage);
615 if (status != DDI_SUCCESS) {
616 /* undo inits done before mod_install */
617 daplka_resource_fini();
618 mutex_destroy(&daplka_dbglock);
619 ddi_soft_state_fini(&daplka_state);
620 }
621 return (status);
622 }
623
624 /*
625 * Module Removal
626 */
627 int
628 _fini(void)
629 {
630 int status;
631
632 /*
633 * mod_remove causes detach to be called
634 */
635 if ((status = mod_remove(&modlinkage)) != 0) {
636 DERR("fini: mod_remove failed: 0x%x\n", status);
637 return (status);
638 }
639
640 daplka_resource_fini();
641 mutex_destroy(&daplka_dbglock);
642 ddi_soft_state_fini(&daplka_state);
643
644 return (status);
645 }
646
647 /*
648 * Return Module Info.
649 */
650 int
651 _info(struct modinfo *modinfop)
652 {
653 return (mod_info(&modlinkage, modinfop));
654 }
655
656 static void
657 daplka_enqueue_hca(daplka_t *dp, daplka_hca_t *hca)
658 {
659 daplka_hca_t *h;
660
661 ASSERT(mutex_owned(&dp->daplka_mutex));
662
663 if (dp->daplka_hca_list_head == NULL) {
664 dp->daplka_hca_list_head = hca;
665 } else {
666 h = dp->daplka_hca_list_head;
667 while (h->hca_next != NULL)
668 h = h->hca_next;
669
670 h->hca_next = hca;
671 }
672 }
673
674 static void
675 daplka_dequeue_hca(daplka_t *dp, daplka_hca_t *hca)
676 {
677 daplka_hca_t *h;
678
679 ASSERT(mutex_owned(&dp->daplka_mutex));
680
681 if (dp->daplka_hca_list_head == hca)
682 dp->daplka_hca_list_head = hca->hca_next;
683 else {
684 h = dp->daplka_hca_list_head;
685 while (h->hca_next != hca)
686 h = h->hca_next;
687 h->hca_next = hca->hca_next;
688 }
689 }
690
691 static int
692 daplka_init_hca(daplka_t *dp, ib_guid_t hca_guid)
693 {
694 daplka_hca_t *hca;
695 ibt_hca_portinfo_t *pinfop;
696 uint_t size;
697 int j;
698 ibt_status_t status;
699
700 hca = kmem_zalloc(sizeof (daplka_hca_t), KM_SLEEP);
701
702 hca->hca_guid = hca_guid;
703
704 /*
705 * open the HCA for use
706 */
707 status = ibt_open_hca(dp->daplka_clnt_hdl, hca_guid, &hca->hca_hdl);
708 if (status != IBT_SUCCESS) {
709 if (status == IBT_HCA_IN_USE) {
710 DERR("ibt_open_hca() returned IBT_HCA_IN_USE\n");
711 } else {
712 DERR("ibt_open_hca() returned %d\n", status);
713 }
714 kmem_free(hca, sizeof (daplka_hca_t));
715 return (status);
716 }
717
718 /*
719 * query HCA to get its info
720 */
721 status = ibt_query_hca(hca->hca_hdl, &hca->hca_attr);
722 if (status != IBT_SUCCESS) {
723 DERR("ibt_query_hca returned %d (hca_guid 0x%llx)\n",
724 status, (longlong_t)hca_guid);
725 goto out;
726 }
727
728 /*
729 * query HCA to get info of all ports
730 */
731 status = ibt_query_hca_ports(hca->hca_hdl,
732 0, &pinfop, &hca->hca_nports, &size);
733 if (status != IBT_SUCCESS) {
734 DERR("ibt_query_all_ports returned %d "
735 "(hca_guid 0x%llx)\n", status,
736 (longlong_t)hca_guid);
737 goto out;
738 }
739 hca->hca_ports = pinfop;
740 hca->hca_pinfosz = size;
741
742 DERR("hca guid 0x%llx, nports %d\n",
743 (longlong_t)hca_guid, hca->hca_nports);
744 for (j = 0; j < hca->hca_nports; j++) {
745 DERR("port %d: state %d prefix 0x%016llx "
746 "guid %016llx\n",
747 pinfop[j].p_port_num, pinfop[j].p_linkstate,
748 (longlong_t)pinfop[j].p_sgid_tbl[0].gid_prefix,
749 (longlong_t)pinfop[j].p_sgid_tbl[0].gid_guid);
750 }
751
752 mutex_enter(&dp->daplka_mutex);
753 daplka_enqueue_hca(dp, hca);
754 mutex_exit(&dp->daplka_mutex);
755
756 return (IBT_SUCCESS);
757
758 out:
759 (void) ibt_close_hca(hca->hca_hdl);
760 kmem_free(hca, sizeof (daplka_hca_t));
761 return (status);
762 }
763
764 /*
765 * this function obtains the list of HCAs from IBTF.
766 * the HCAs are then opened and the returned handles
767 * and attributes are stored into the global daplka_dev
768 * structure.
769 */
770 static int
771 daplka_init_hcas(daplka_t *dp)
772 {
773 int i;
774 ib_guid_t *hca_guids;
775 uint32_t hca_count;
776
777 /*
778 * get the num & list of HCAs present
779 */
780 hca_count = ibt_get_hca_list(&hca_guids);
781 DERR("No. of HCAs present %d\n", hca_count);
782
783 if (hca_count != 0) {
784 /*
785 * get the info for each available HCA
786 */
787 for (i = 0; i < hca_count; i++)
788 (void) daplka_init_hca(dp, hca_guids[i]);
789
790 ibt_free_hca_list(hca_guids, hca_count);
791 }
792
793 if (dp->daplka_hca_list_head != NULL)
794 return (IBT_SUCCESS);
795 else
796 return (IBT_FAILURE);
797 }
798
799 static int
800 daplka_fini_hca(daplka_t *dp, daplka_hca_t *hca)
801 {
802 ibt_status_t status;
803
804 if (hca->hca_hdl != NULL) {
805 status = ibt_close_hca(hca->hca_hdl);
806 if (status != IBT_SUCCESS) {
807 DERR("ibt_close_hca returned %d"
808 " (hca_guid 0x%llx)\n", status,
809 (longlong_t)hca->hca_guid);
810
811 mutex_enter(&dp->daplka_mutex);
812 daplka_enqueue_hca(dp, hca);
813 mutex_exit(&dp->daplka_mutex);
814
815 return (status);
816 }
817 }
818
819 if (hca->hca_ports != NULL)
820 ibt_free_portinfo(hca->hca_ports, hca->hca_pinfosz);
821
822 kmem_free(hca, sizeof (daplka_hca_t));
823 return (IBT_SUCCESS);
824 }
825
826 /*
827 * closes all HCAs and frees up the HCA list
828 */
829 static int
830 daplka_fini_hcas(daplka_t *dp)
831 {
832 ibt_status_t status;
833 daplka_hca_t *hca;
834
835 mutex_enter(&daplka_dev->daplka_mutex);
836 while ((hca = dp->daplka_hca_list_head) != NULL) {
837 if (DAPLKA_HCA_BUSY(hca)) {
838 mutex_exit(&daplka_dev->daplka_mutex);
839 return (IBT_HCA_RESOURCES_NOT_FREED);
840 }
841 daplka_dequeue_hca(daplka_dev, hca);
842 mutex_exit(&daplka_dev->daplka_mutex);
843
844 if ((status = daplka_fini_hca(dp, hca)) != IBT_SUCCESS)
845 return (status);
846
847 mutex_enter(&daplka_dev->daplka_mutex);
848 }
849 mutex_exit(&daplka_dev->daplka_mutex);
850
851 DERR("dapl kernel agent unloaded\n");
852 return (IBT_SUCCESS);
853 }
854
855
856 /*
857 * Attach the device, create and fill in daplka_dev
858 */
859 static int
860 daplka_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
861 {
862 daplka_t *dp;
863 int instance, retval, err;
864 boolean_t sp_htbl_allocated = B_FALSE;
865 boolean_t timer_htbl_allocated = B_FALSE;
866 boolean_t shared_mr_tree_allocated = B_FALSE;
867
868 switch (cmd) {
869 case DDI_ATTACH:
870 break;
871 case DDI_RESUME:
872 return (DDI_SUCCESS);
873 default:
874 return (DDI_FAILURE);
875 }
876
877 /*
878 * Allocate soft data structure
879 */
880 instance = ddi_get_instance(dip);
881 if (ddi_soft_state_zalloc(daplka_state, instance) != DDI_SUCCESS) {
882 DERR("attach: bad state zalloc\n");
883 return (DDI_FAILURE);
884 }
885
886 dp = ddi_get_soft_state(daplka_state, instance);
887 if (dp == NULL) {
888 ddi_soft_state_free(daplka_state, instance);
889 DERR("attach: cannot get soft state\n");
890 return (DDI_FAILURE);
891 }
892 /*
893 * Stuff private info into dip.
894 */
895 dp->daplka_dip = dip;
896 ddi_set_driver_private(dip, dp);
897 daplka_dev = dp;
898 mutex_init(&dp->daplka_mutex, NULL, MUTEX_DRIVER, NULL);
899
900 /*
901 * Register driver with IBTF
902 */
903 retval = ibt_attach(&daplka_clnt_modinfo, dip, dp,
904 &dp->daplka_clnt_hdl);
905 if (retval != IBT_SUCCESS) {
906 DERR("attach: ibt_attach failed: error = %d\n", retval);
907 retval = DDI_FAILURE;
908 goto error;
909 }
910 /* Register to receive SM events */
911 ibt_register_subnet_notices(dp->daplka_clnt_hdl,
912 daplka_sm_notice_handler, NULL);
913
914 retval = daplka_init_hcas(dp);
915 if (retval != IBT_SUCCESS) {
916 DERR("attach: hca_init failed: error = %d\n", retval);
917 retval = DDI_FAILURE;
918 goto error;
919 }
920 /*
921 * this table is used by cr_handoff
922 */
923 retval = daplka_hash_create(&daplka_global_sp_htbl,
924 DAPLKA_G_SP_HTBL_SZ, daplka_hash_sp_unref,
925 daplka_hash_generic_lookup);
926 if (retval != 0) {
927 DERR("attach: cannot create sp hash table\n");
928 retval = DDI_FAILURE;
929 goto error;
930 }
931 sp_htbl_allocated = B_TRUE;
932
933 /*
934 * this table stores per EP timer information.
935 * timer_info_t objects are inserted into this table whenever
936 * a EP timer is set. timers get removed when they expire
937 * or when they get cancelled.
938 */
939 retval = daplka_hash_create(&daplka_timer_info_htbl,
940 DAPLKA_TIMER_HTBL_SZ, daplka_hash_timer_free, NULL);
941 if (retval != 0) {
942 DERR("attach: cannot create timer hash table\n");
943 retval = DDI_FAILURE;
944 goto error;
945 }
946 timer_htbl_allocated = B_TRUE;
947
948 /*
949 * this taskq is currently only used for processing timers.
950 * other processing may also use this taskq in the future.
951 */
952 daplka_taskq = taskq_create(DAPLKA_DRV_NAME, DAPLKA_TQ_NTHREADS,
953 maxclsyspri, 1, DAPLKA_TQ_NTHREADS, TASKQ_DYNAMIC);
954 if (daplka_taskq == NULL) {
955 DERR("attach: cannot create daplka_taskq\n");
956 retval = DDI_FAILURE;
957 goto error;
958 }
959
960 /*
961 * daplka_shared_mr_tree holds daplka_shared_mr_t objects that
962 * gets retrieved or created when daplka_mr_register_shared is
963 * called.
964 */
965 mutex_init(&daplka_shared_mr_lock, NULL, MUTEX_DRIVER, NULL);
966
967 avl_create(&daplka_shared_mr_tree, daplka_shared_mr_cmp,
968 sizeof (daplka_shared_mr_t),
969 offsetof(daplka_shared_mr_t, smr_node));
970 shared_mr_tree_allocated = B_TRUE;
971
972 /*
973 * Create the filesystem device node.
974 */
975 if (ddi_create_minor_node(dip, DAPLKA_MINOR_NAME, S_IFCHR,
976 0, DDI_PSEUDO, 0) != DDI_SUCCESS) {
977 DERR("attach: bad create_minor_node\n");
978 retval = DDI_FAILURE;
979 goto error;
980 }
981 dp->daplka_status = DAPLKA_STATE_ATTACHED;
982 ddi_report_dev(dip);
983 return (DDI_SUCCESS);
984
985 error:
986 if (shared_mr_tree_allocated) {
987 avl_destroy(&daplka_shared_mr_tree);
988 mutex_destroy(&daplka_shared_mr_lock);
989 }
990
991 if (daplka_taskq) {
992 taskq_destroy(daplka_taskq);
993 daplka_taskq = NULL;
994 }
995
996 if (timer_htbl_allocated) {
997 daplka_hash_destroy(&daplka_timer_info_htbl);
998 }
999
1000 if (sp_htbl_allocated) {
1001 daplka_hash_destroy(&daplka_global_sp_htbl);
1002 }
1003
1004 err = daplka_fini_hcas(dp);
1005 if (err != IBT_SUCCESS) {
1006 DERR("attach: hca_fini returned %d\n", err);
1007 }
1008
1009 if (dp->daplka_clnt_hdl != NULL) {
1010 /* unregister SM event notification */
1011 ibt_register_subnet_notices(dp->daplka_clnt_hdl,
1012 (ibt_sm_notice_handler_t)NULL, NULL);
1013 err = ibt_detach(dp->daplka_clnt_hdl);
1014
1015 if (err != IBT_SUCCESS) {
1016 DERR("attach: ibt_detach returned %d\n", err);
1017 }
1018 }
1019 mutex_destroy(&dp->daplka_mutex);
1020
1021 if (dp->daplka_status == DAPLKA_STATE_ATTACHED) {
1022 ddi_remove_minor_node(dip, NULL);
1023 }
1024 ddi_soft_state_free(daplka_state, instance);
1025 return (retval);
1026 }
1027
1028 /*
1029 * Detach - Free resources allocated in attach
1030 */
1031 /* ARGSUSED */
1032 static int
1033 daplka_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1034 {
1035 int instance, err;
1036 void *cookie = NULL;
1037 daplka_t *dp;
1038
1039 if (cmd != DDI_DETACH) {
1040 return (DDI_FAILURE);
1041 }
1042 if (daplka_resource.daplka_rc_cnt > 0 ||
1043 daplka_pending_close > 0) {
1044 DERR("detach: driver in use\n");
1045 return (DDI_FAILURE);
1046 }
1047
1048 instance = ddi_get_instance(dip);
1049 dp = ddi_get_soft_state(daplka_state, instance);
1050 if (dp == NULL) {
1051 DERR("detach: cannot get soft state\n");
1052 return (DDI_FAILURE);
1053 }
1054 err = daplka_fini_hcas(dp);
1055 if (err != IBT_SUCCESS) {
1056 DERR("detach: hca_fini returned %d\n", err);
1057 return (DDI_FAILURE);
1058 }
1059 if (dp->daplka_clnt_hdl != NULL) {
1060 /* unregister SM event notification */
1061 ibt_register_subnet_notices(dp->daplka_clnt_hdl,
1062 (ibt_sm_notice_handler_t)NULL, NULL);
1063 err = ibt_detach(dp->daplka_clnt_hdl);
1064 if (err != IBT_SUCCESS) {
1065 DERR("detach: ibt_detach returned %d\n", err);
1066 return (DDI_FAILURE);
1067 }
1068 dp->daplka_clnt_hdl = NULL;
1069 }
1070 mutex_destroy(&dp->daplka_mutex);
1071 if (dp->daplka_status == DAPLKA_STATE_ATTACHED) {
1072 ddi_remove_minor_node(dip, NULL);
1073 }
1074 dp->daplka_status = DAPLKA_STATE_DETACHED;
1075 ddi_soft_state_free(daplka_state, instance);
1076 daplka_dev = NULL;
1077
1078 /*
1079 * by the time we get here, all clients of dapl should
1080 * have exited and completed their cleanup properly.
1081 * we can assert that all global data structures are now
1082 * empty.
1083 */
1084 ASSERT(avl_destroy_nodes(&daplka_shared_mr_tree, &cookie) == NULL);
1085 avl_destroy(&daplka_shared_mr_tree);
1086 mutex_destroy(&daplka_shared_mr_lock);
1087
1088 ASSERT(daplka_hash_getsize(&daplka_timer_info_htbl) == 0);
1089 daplka_hash_destroy(&daplka_timer_info_htbl);
1090
1091 ASSERT(daplka_hash_getsize(&daplka_global_sp_htbl) == 0);
1092 daplka_hash_destroy(&daplka_global_sp_htbl);
1093
1094 taskq_destroy(daplka_taskq);
1095
1096 return (DDI_SUCCESS);
1097 }
1098
1099 /* ARGSUSED */
1100 static int
1101 daplka_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1102 {
1103 switch (infocmd) {
1104 case DDI_INFO_DEVT2DEVINFO:
1105 if (daplka_dev != NULL) {
1106 *result = daplka_dev->daplka_dip;
1107 return (DDI_SUCCESS);
1108 } else {
1109 return (DDI_FAILURE);
1110 }
1111
1112 case DDI_INFO_DEVT2INSTANCE:
1113 *result = 0;
1114 return (DDI_SUCCESS);
1115
1116 default:
1117 return (DDI_FAILURE);
1118 }
1119 }
1120
1121 /*
1122 * creates a EP resource.
1123 * A EP resource contains a RC channel. A EP resource holds a
1124 * reference to a send_evd (for the send CQ), recv_evd (for the
1125 * recv CQ), a connection evd and a PD. These references ensure
1126 * that the referenced resources are not freed until the EP itself
1127 * gets freed.
1128 */
1129 /* ARGSUSED */
1130 static int
1131 daplka_ep_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
1132 cred_t *cred, int *rvalp)
1133 {
1134 daplka_ep_resource_t *ep_rp;
1135 daplka_pd_resource_t *pd_rp;
1136 dapl_ep_create_t args;
1137 ibt_rc_chan_alloc_args_t chan_args;
1138 ibt_chan_alloc_flags_t achan_flags;
1139 ibt_chan_sizes_t chan_real_sizes;
1140 ibt_hca_attr_t *hca_attrp;
1141 uint64_t ep_hkey = 0;
1142 boolean_t inserted = B_FALSE;
1143 uint32_t old_state, new_state;
1144 int retval;
1145 ibt_status_t status;
1146
1147 D3("ep_create: enter\n");
1148 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_create_t),
1149 mode);
1150 if (retval != 0) {
1151 DERR("ep_create: copyin error %d\n", retval);
1152 return (EFAULT);
1153 }
1154 ep_rp = kmem_zalloc(sizeof (daplka_ep_resource_t), daplka_km_flags);
1155 if (ep_rp == NULL) {
1156 DERR("ep_create: cannot allocate ep_rp\n");
1157 return (ENOMEM);
1158 }
1159 DAPLKA_RS_INIT(ep_rp, DAPL_TYPE_EP,
1160 DAPLKA_RS_RNUM(ia_rp), daplka_ep_destroy);
1161
1162 mutex_init(&ep_rp->ep_lock, NULL, MUTEX_DRIVER, NULL);
1163 cv_init(&ep_rp->ep_cv, NULL, CV_DRIVER, NULL);
1164 ep_rp->ep_hca = ia_rp->ia_hca;
1165 ep_rp->ep_cookie = args.ep_cookie;
1166 ep_rp->ep_timer_hkey = 0;
1167
1168 /*
1169 * we don't have to use ep_get_state here because ep_rp is not in
1170 * ep_htbl yet. refer to the description of daplka_ep_set_state
1171 * for details about the EP state machine.
1172 */
1173 ep_rp->ep_state = DAPLKA_EP_STATE_TRANSITIONING;
1174 new_state = old_state = DAPLKA_EP_STATE_CLOSED;
1175
1176 /* get reference to send evd and get cq handle */
1177 ep_rp->ep_snd_evd = (daplka_evd_resource_t *)
1178 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_snd_evd_hkey);
1179 if (ep_rp->ep_snd_evd == NULL) {
1180 DERR("ep_create: ep_snd_evd %llx not found\n",
1181 args.ep_snd_evd_hkey);
1182 retval = EINVAL;
1183 goto cleanup;
1184 }
1185 chan_args.rc_scq = ep_rp->ep_snd_evd->evd_cq_hdl;
1186 if (chan_args.rc_scq == NULL) {
1187 DERR("ep_create: ep_snd_evd cq invalid\n");
1188 retval = EINVAL;
1189 goto cleanup;
1190 }
1191
1192 /* get reference to recv evd and get cq handle */
1193 ep_rp->ep_rcv_evd = (daplka_evd_resource_t *)
1194 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_rcv_evd_hkey);
1195 if (ep_rp->ep_rcv_evd == NULL) {
1196 DERR("ep_create: ep_rcv_evd %llx not found\n",
1197 args.ep_rcv_evd_hkey);
1198 retval = EINVAL;
1199 goto cleanup;
1200 }
1201 chan_args.rc_rcq = ep_rp->ep_rcv_evd->evd_cq_hdl;
1202 if (chan_args.rc_rcq == NULL) {
1203 DERR("ep_create: ep_rcv_evd cq invalid\n");
1204 retval = EINVAL;
1205 goto cleanup;
1206 }
1207
1208 /* get reference to conn evd */
1209 ep_rp->ep_conn_evd = (daplka_evd_resource_t *)
1210 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_conn_evd_hkey);
1211 if (ep_rp->ep_conn_evd == NULL) {
1212 DERR("ep_create: ep_conn_evd %llx not found\n",
1213 args.ep_conn_evd_hkey);
1214 retval = EINVAL;
1215 goto cleanup;
1216 }
1217
1218 /* get reference to SRQ if needed */
1219 if (args.ep_srq_attached) {
1220 ep_rp->ep_srq_res = (daplka_srq_resource_t *)daplka_hash_lookup(
1221 &ia_rp->ia_srq_htbl, args.ep_srq_hkey);
1222 if (ep_rp->ep_srq_res == NULL) {
1223 DERR("ep_create: ep_srq %llx not found\n",
1224 (longlong_t)args.ep_srq_hkey);
1225 retval = EINVAL;
1226 goto cleanup;
1227 }
1228 ASSERT(DAPLKA_RS_TYPE(ep_rp->ep_srq_res) == DAPL_TYPE_SRQ);
1229 D3("ep_create: ep_srq %p %llx\n", ep_rp->ep_srq_res,
1230 (longlong_t)args.ep_srq_hkey);
1231 } else {
1232 ep_rp->ep_srq_res = NULL;
1233 }
1234
1235 /* get pd handle */
1236 pd_rp = (daplka_pd_resource_t *)
1237 daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.ep_pd_hkey);
1238 if (pd_rp == NULL) {
1239 DERR("ep_create: cannot find pd resource\n");
1240 retval = EINVAL;
1241 goto cleanup;
1242 }
1243 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
1244 ep_rp->ep_pd_res = pd_rp;
1245 chan_args.rc_pd = pd_rp->pd_hdl;
1246
1247
1248 /*
1249 * these checks ensure that the requested channel sizes
1250 * are within the limits supported by the chosen HCA.
1251 */
1252 hca_attrp = &ia_rp->ia_hca->hca_attr;
1253 if (args.ep_ch_sizes.dcs_sq_sgl > hca_attrp->hca_max_sgl) {
1254 DERR("ep_create: invalid cs_sq_sgl %d\n",
1255 args.ep_ch_sizes.dcs_sq_sgl);
1256 retval = EINVAL;
1257 goto cleanup;
1258 }
1259 if (args.ep_ch_sizes.dcs_rq_sgl > hca_attrp->hca_max_sgl) {
1260 DERR("ep_create: invalid cs_rq_sgl %d\n",
1261 args.ep_ch_sizes.dcs_rq_sgl);
1262 retval = EINVAL;
1263 goto cleanup;
1264 }
1265 if (args.ep_ch_sizes.dcs_sq > hca_attrp->hca_max_chan_sz) {
1266 DERR("ep_create: invalid cs_sq %d\n",
1267 args.ep_ch_sizes.dcs_sq);
1268 retval = EINVAL;
1269 goto cleanup;
1270 }
1271 if (args.ep_ch_sizes.dcs_rq > hca_attrp->hca_max_chan_sz) {
1272 DERR("ep_create: invalid cs_rq %d\n",
1273 args.ep_ch_sizes.dcs_rq);
1274 retval = EINVAL;
1275 goto cleanup;
1276 }
1277
1278 chan_args.rc_sizes.cs_sq_sgl = args.ep_ch_sizes.dcs_sq_sgl;
1279 chan_args.rc_sizes.cs_rq_sgl = args.ep_ch_sizes.dcs_rq_sgl;
1280 chan_args.rc_sizes.cs_sq = args.ep_ch_sizes.dcs_sq;
1281 chan_args.rc_sizes.cs_rq = args.ep_ch_sizes.dcs_rq;
1282 chan_args.rc_flags = IBT_WR_SIGNALED;
1283 chan_args.rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
1284 chan_args.rc_hca_port_num = ia_rp->ia_port_num;
1285 chan_args.rc_clone_chan = NULL;
1286 if (args.ep_srq_attached) {
1287 chan_args.rc_srq = ep_rp->ep_srq_res->srq_hdl;
1288 } else {
1289 chan_args.rc_srq = NULL;
1290 }
1291
1292 D3("ep_create: sq_sgl %d, rq_sgl %d, sq %d, rq %d, "
1293 "sig_type 0x%x, control 0x%x, portnum %d, clone_chan 0x%p\n",
1294 args.ep_ch_sizes.dcs_sq_sgl, args.ep_ch_sizes.dcs_rq_sgl,
1295 args.ep_ch_sizes.dcs_sq, args.ep_ch_sizes.dcs_rq,
1296 chan_args.rc_flags, chan_args.rc_control,
1297 chan_args.rc_hca_port_num, chan_args.rc_clone_chan);
1298
1299 if (args.ep_srq_attached) {
1300 achan_flags = IBT_ACHAN_USER_MAP | IBT_ACHAN_USES_SRQ;
1301 } else {
1302 achan_flags = IBT_ACHAN_USER_MAP;
1303 }
1304 /* create rc channel */
1305 status = daplka_ibt_alloc_rc_channel(ep_rp, ia_rp->ia_hca_hdl,
1306 achan_flags, &chan_args, &ep_rp->ep_chan_hdl,
1307 &chan_real_sizes);
1308 if (status != IBT_SUCCESS) {
1309 DERR("ep_create: alloc_rc_channel returned %d\n", status);
1310 *rvalp = (int)status;
1311 retval = 0;
1312 goto cleanup;
1313 }
1314
1315 args.ep_ch_real_sizes.dcs_sq = chan_real_sizes.cs_sq;
1316 args.ep_ch_real_sizes.dcs_rq = chan_real_sizes.cs_rq;
1317 args.ep_ch_real_sizes.dcs_sq_sgl = chan_real_sizes.cs_sq_sgl;
1318 args.ep_ch_real_sizes.dcs_rq_sgl = chan_real_sizes.cs_rq_sgl;
1319
1320 /*
1321 * store ep ptr with chan_hdl.
1322 * this ep_ptr is used by the CM handlers (both active and
1323 * passive)
1324 * mutex is only needed for race of "destroy" and "async"
1325 */
1326 mutex_enter(&daplka_dev->daplka_mutex);
1327 ibt_set_chan_private(ep_rp->ep_chan_hdl, (void *)ep_rp);
1328 mutex_exit(&daplka_dev->daplka_mutex);
1329
1330 /* Get HCA-specific data_out info */
1331 status = ibt_ci_data_out(ia_rp->ia_hca_hdl,
1332 IBT_CI_NO_FLAGS, IBT_HDL_CHANNEL, (void *)ep_rp->ep_chan_hdl,
1333 &args.ep_qp_data_out, sizeof (args.ep_qp_data_out));
1334
1335 if (status != IBT_SUCCESS) {
1336 DERR("ep_create: ibt_ci_data_out error(%d)\n",
1337 status);
1338 *rvalp = (int)status;
1339 retval = 0;
1340 goto cleanup;
1341 }
1342
1343 /* insert into ep hash table */
1344 retval = daplka_hash_insert(&ia_rp->ia_ep_htbl,
1345 &ep_hkey, (void *)ep_rp);
1346 if (retval != 0) {
1347 DERR("ep_create: cannot insert ep resource into ep_htbl\n");
1348 goto cleanup;
1349 }
1350 inserted = B_TRUE;
1351
1352 /*
1353 * at this point, the ep_rp can be looked up by other threads
1354 * if they manage to guess the correct hkey. but they are not
1355 * permitted to operate on ep_rp until we transition to the
1356 * CLOSED state.
1357 */
1358
1359 /* return hkey to library */
1360 args.ep_hkey = ep_hkey;
1361
1362 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_ep_create_t),
1363 mode);
1364 if (retval != 0) {
1365 DERR("ep_create: copyout error %d\n", retval);
1366 retval = EFAULT;
1367 goto cleanup;
1368 }
1369
1370 daplka_ep_set_state(ep_rp, old_state, new_state);
1371 D3("ep_create: exit\n");
1372 return (0);
1373
1374 cleanup:
1375 if (inserted) {
1376 daplka_ep_resource_t *free_rp = NULL;
1377
1378 (void) daplka_hash_remove(&ia_rp->ia_ep_htbl, ep_hkey,
1379 (void **)&free_rp);
1380 if (free_rp != ep_rp) {
1381 /*
1382 * this case is impossible because ep_free will
1383 * wait until our state transition is complete.
1384 */
1385 DERR("ep_create: cannot remove ep from hash table\n");
1386 ASSERT(B_FALSE);
1387 return (retval);
1388 }
1389 }
1390 new_state = DAPLKA_EP_STATE_FREED;
1391 daplka_ep_set_state(ep_rp, old_state, new_state);
1392 DAPLKA_RS_UNREF(ep_rp);
1393 return (retval);
1394 }
1395
1396 /*
1397 * daplka_ep_get_state retrieves the current state of the EP and
1398 * sets the state to TRANSITIONING. if the current state is already
1399 * TRANSITIONING, this function will wait until the state becomes one
1400 * of the other EP states. Most of the EP related ioctls follow the
1401 * call sequence:
1402 *
1403 * new_state = old_state = daplka_ep_get_state(ep_rp);
1404 * ...
1405 * ...some code that affects the EP
1406 * ...
1407 * new_state = <NEW_STATE>;
1408 * daplka_ep_set_state(ep_rp, old_state, new_state);
1409 *
1410 * this call sequence ensures that only one thread may access the EP
1411 * during the time ep_state is in TRANSITIONING. daplka_ep_set_state
1412 * transitions ep_state to new_state and wakes up any waiters blocking
1413 * on ep_cv.
1414 *
1415 */
1416 static uint32_t
1417 daplka_ep_get_state(daplka_ep_resource_t *ep_rp)
1418 {
1419 uint32_t old_state = 0;
1420
1421 mutex_enter(&ep_rp->ep_lock);
1422 while (ep_rp->ep_state == DAPLKA_EP_STATE_TRANSITIONING) {
1423 D2("get_state: wait for state transition to complete\n");
1424 cv_wait(&ep_rp->ep_cv, &ep_rp->ep_lock);
1425 D2("get_state: done, curr state = %d\n", ep_rp->ep_state);
1426 }
1427 ASSERT(ep_rp->ep_state != DAPLKA_EP_STATE_TRANSITIONING);
1428 old_state = ep_rp->ep_state;
1429
1430 /*
1431 * an ep that is in the FREED state cannot transition
1432 * back to any of the regular states
1433 */
1434 if (old_state != DAPLKA_EP_STATE_FREED) {
1435 ep_rp->ep_state = DAPLKA_EP_STATE_TRANSITIONING;
1436 }
1437 mutex_exit(&ep_rp->ep_lock);
1438 return (old_state);
1439 }
1440
1441 /*
1442 * EP state transition diagram
1443 *
1444 * CLOSED<-------------------
1445 * | |
1446 * | |
1447 * ------------------------ |
1448 * | | |
1449 * | | |
1450 * v v |
1451 * CONNECTING ACCEPTING |
1452 * | | | | | |
1453 * | | | | | |
1454 * | | | | | |
1455 * | | |_______|_______| |
1456 * | | | | | |
1457 * | |___________| | | |
1458 * | | | | |
1459 * | v | |---->DISCONNECTED
1460 * | CONNECTED | ^
1461 * v | | |
1462 * ABORTING |---------|--------------|
1463 * | | | |
1464 * | | v |
1465 * | |-------->DISCONNECTING--|
1466 * | |
1467 * |---------------------------------|
1468 *
1469 * *not shown in this diagram:
1470 * -loopback transitions
1471 * -transitions to the FREED state
1472 */
1473 static boolean_t
1474 daplka_ep_transition_is_valid(uint32_t old_state, uint32_t new_state)
1475 {
1476 boolean_t valid = B_FALSE;
1477
1478 /*
1479 * reseting to the same state is a no-op and is always
1480 * permitted. transitioning to the FREED state indicates
1481 * that the ep is about to be freed and no further operation
1482 * is allowed on it. to support abrupt close, the ep is
1483 * permitted to transition to the FREED state from any state.
1484 */
1485 if (old_state == new_state ||
1486 new_state == DAPLKA_EP_STATE_FREED) {
1487 return (B_TRUE);
1488 }
1489
1490 switch (old_state) {
1491 case DAPLKA_EP_STATE_CLOSED:
1492 /*
1493 * this is the initial ep_state.
1494 * a transition to CONNECTING or ACCEPTING may occur
1495 * upon calling daplka_ep_connect or daplka_cr_accept,
1496 * respectively.
1497 */
1498 if (new_state == DAPLKA_EP_STATE_CONNECTING ||
1499 new_state == DAPLKA_EP_STATE_ACCEPTING) {
1500 valid = B_TRUE;
1501 }
1502 break;
1503 case DAPLKA_EP_STATE_CONNECTING:
1504 /*
1505 * we transition to this state if daplka_ep_connect
1506 * is successful. from this state, we can transition
1507 * to CONNECTED if daplka_cm_rc_conn_est gets called;
1508 * or to DISCONNECTED if daplka_cm_rc_conn_closed or
1509 * daplka_cm_rc_event_failure gets called. If the
1510 * client calls daplka_ep_disconnect, we transition
1511 * to DISCONNECTING. If a timer was set at ep_connect
1512 * time and if the timer expires prior to any of the
1513 * CM callbacks, we transition to ABORTING and then
1514 * to DISCONNECTED.
1515 */
1516 if (new_state == DAPLKA_EP_STATE_CONNECTED ||
1517 new_state == DAPLKA_EP_STATE_DISCONNECTING ||
1518 new_state == DAPLKA_EP_STATE_DISCONNECTED ||
1519 new_state == DAPLKA_EP_STATE_ABORTING) {
1520 valid = B_TRUE;
1521 }
1522 break;
1523 case DAPLKA_EP_STATE_ACCEPTING:
1524 /*
1525 * we transition to this state if daplka_cr_accept
1526 * is successful. from this state, we can transition
1527 * to CONNECTED if daplka_cm_service_conn_est gets called;
1528 * or to DISCONNECTED if daplka_cm_service_conn_closed or
1529 * daplka_cm_service_event_failure gets called. If the
1530 * client calls daplka_ep_disconnect, we transition to
1531 * DISCONNECTING.
1532 */
1533 if (new_state == DAPLKA_EP_STATE_CONNECTED ||
1534 new_state == DAPLKA_EP_STATE_DISCONNECTING ||
1535 new_state == DAPLKA_EP_STATE_DISCONNECTED) {
1536 valid = B_TRUE;
1537 }
1538 break;
1539 case DAPLKA_EP_STATE_CONNECTED:
1540 /*
1541 * we transition to this state if a active or passive
1542 * connection gets established. if the client calls
1543 * daplka_ep_disconnect, we transition to the
1544 * DISCONNECTING state. subsequent CM callbacks will
1545 * cause ep_state to be set to DISCONNECTED. If the
1546 * remote peer terminates the connection before we do,
1547 * it is possible for us to transition directly from
1548 * CONNECTED to DISCONNECTED.
1549 */
1550 if (new_state == DAPLKA_EP_STATE_DISCONNECTING ||
1551 new_state == DAPLKA_EP_STATE_DISCONNECTED) {
1552 valid = B_TRUE;
1553 }
1554 break;
1555 case DAPLKA_EP_STATE_DISCONNECTING:
1556 /*
1557 * we transition to this state if the client calls
1558 * daplka_ep_disconnect.
1559 */
1560 if (new_state == DAPLKA_EP_STATE_DISCONNECTED) {
1561 valid = B_TRUE;
1562 }
1563 break;
1564 case DAPLKA_EP_STATE_ABORTING:
1565 /*
1566 * we transition to this state if the active side
1567 * EP timer has expired. this is only a transient
1568 * state that is set during timer processing. when
1569 * timer processing completes, ep_state will become
1570 * DISCONNECTED.
1571 */
1572 if (new_state == DAPLKA_EP_STATE_DISCONNECTED) {
1573 valid = B_TRUE;
1574 }
1575 break;
1576 case DAPLKA_EP_STATE_DISCONNECTED:
1577 /*
1578 * we transition to this state if we get a closed
1579 * or event_failure CM callback. an expired timer
1580 * can also cause us to be in this state. this
1581 * is the only state in which we permit the
1582 * ep_reinit operation.
1583 */
1584 if (new_state == DAPLKA_EP_STATE_CLOSED) {
1585 valid = B_TRUE;
1586 }
1587 break;
1588 default:
1589 break;
1590 }
1591
1592 if (!valid) {
1593 DERR("ep_transition: invalid state change %d -> %d\n",
1594 old_state, new_state);
1595 }
1596 return (valid);
1597 }
1598
1599 /*
1600 * first check if the transition is valid. then set ep_state
1601 * to new_state and wake up all waiters.
1602 */
1603 static void
1604 daplka_ep_set_state(daplka_ep_resource_t *ep_rp, uint32_t old_state,
1605 uint32_t new_state)
1606 {
1607 boolean_t valid;
1608
1609 ASSERT(new_state != DAPLKA_EP_STATE_TRANSITIONING);
1610
1611 valid = daplka_ep_transition_is_valid(old_state, new_state);
1612 mutex_enter(&ep_rp->ep_lock);
1613 if (ep_rp->ep_state != DAPLKA_EP_STATE_FREED) {
1614 if (valid) {
1615 ep_rp->ep_state = new_state;
1616 } else {
1617 /*
1618 * this case is impossible.
1619 * we have a serious problem if we get here.
1620 * instead of panicing, we reset the state to
1621 * old_state. doing this would at least prevent
1622 * threads from hanging due to ep_state being
1623 * stuck in TRANSITIONING.
1624 */
1625 ep_rp->ep_state = old_state;
1626 ASSERT(B_FALSE);
1627 }
1628 }
1629 cv_broadcast(&ep_rp->ep_cv);
1630 mutex_exit(&ep_rp->ep_lock);
1631 }
1632
1633 /*
1634 * modifies RC channel attributes.
1635 * currently, only the rdma_in and rdma_out attributes may
1636 * be modified. the channel must be in quiescent state when
1637 * this function is called.
1638 */
1639 /* ARGSUSED */
1640 static int
1641 daplka_ep_modify(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
1642 cred_t *cred, int *rvalp)
1643 {
1644 daplka_ep_resource_t *ep_rp = NULL;
1645 ibt_cep_modify_flags_t good_flags;
1646 ibt_rc_chan_modify_attr_t rcm_attr;
1647 ibt_hca_attr_t *hca_attrp;
1648 dapl_ep_modify_t args;
1649 ibt_status_t status;
1650 uint32_t old_state, new_state;
1651 int retval = 0;
1652
1653 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_modify_t),
1654 mode);
1655 if (retval != 0) {
1656 DERR("ep_modify: copyin error %d\n", retval);
1657 return (EFAULT);
1658 }
1659 ep_rp = (daplka_ep_resource_t *)
1660 daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epm_hkey);
1661 if (ep_rp == NULL) {
1662 DERR("ep_modify: cannot find ep resource\n");
1663 return (EINVAL);
1664 }
1665 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
1666 new_state = old_state = daplka_ep_get_state(ep_rp);
1667
1668 if (old_state != DAPLKA_EP_STATE_CLOSED &&
1669 old_state != DAPLKA_EP_STATE_DISCONNECTED) {
1670 DERR("ep_modify: invalid state %d\n", old_state);
1671 retval = EINVAL;
1672 goto cleanup;
1673 }
1674
1675 good_flags = IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_RDMARA_IN;
1676 if ((args.epm_flags & ~good_flags) != 0) {
1677 DERR("ep_modify: invalid flags 0x%x\n", args.epm_flags);
1678 retval = EINVAL;
1679 goto cleanup;
1680 }
1681
1682 hca_attrp = &ia_rp->ia_hca->hca_attr;
1683
1684 bzero(&rcm_attr, sizeof (ibt_rc_chan_modify_attr_t));
1685 if ((args.epm_flags & IBT_CEP_SET_RDMARA_OUT) != 0) {
1686 if (args.epm_rdma_ra_out > hca_attrp->hca_max_rdma_out_chan) {
1687 DERR("ep_modify: invalid epm_rdma_ra_out %d\n",
1688 args.epm_rdma_ra_out);
1689 retval = EINVAL;
1690 goto cleanup;
1691 }
1692 rcm_attr.rc_rdma_ra_out = args.epm_rdma_ra_out;
1693 }
1694 if ((args.epm_flags & IBT_CEP_SET_RDMARA_IN) != 0) {
1695 if (args.epm_rdma_ra_in > hca_attrp->hca_max_rdma_in_chan) {
1696 DERR("ep_modify: epm_rdma_ra_in %d\n",
1697 args.epm_rdma_ra_in);
1698 retval = EINVAL;
1699 goto cleanup;
1700 }
1701 rcm_attr.rc_rdma_ra_in = args.epm_rdma_ra_in;
1702 }
1703 status = ibt_modify_rc_channel(ep_rp->ep_chan_hdl, args.epm_flags,
1704 &rcm_attr, NULL);
1705 if (status != IBT_SUCCESS) {
1706 DERR("ep_modify: modify_rc_channel returned %d\n", status);
1707 *rvalp = (int)status;
1708 retval = 0;
1709 goto cleanup;
1710 }
1711
1712 /*
1713 * ep_modify does not change ep_state
1714 */
1715 cleanup:;
1716 daplka_ep_set_state(ep_rp, old_state, new_state);
1717 DAPLKA_RS_UNREF(ep_rp);
1718 return (retval);
1719 }
1720
1721 /*
1722 * Frees a EP resource.
1723 * a EP may only be freed when it is in the CLOSED or
1724 * DISCONNECTED state.
1725 */
1726 /* ARGSUSED */
1727 static int
1728 daplka_ep_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
1729 cred_t *cred, int *rvalp)
1730 {
1731 daplka_ep_resource_t *ep_rp = NULL;
1732 dapl_ep_free_t args;
1733 uint32_t old_state, new_state;
1734 int retval;
1735
1736 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_free_t), mode);
1737 if (retval != 0) {
1738 DERR("ep_free: copyin error %d\n", retval);
1739 return (EFAULT);
1740 }
1741 ep_rp = (daplka_ep_resource_t *)
1742 daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epf_hkey);
1743 if (ep_rp == NULL) {
1744 DERR("ep_free: cannot find ep resource\n");
1745 return (EINVAL);
1746 }
1747 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
1748 new_state = old_state = daplka_ep_get_state(ep_rp);
1749
1750 /*
1751 * ep cannot be freed if it is in an invalid state.
1752 */
1753 if (old_state != DAPLKA_EP_STATE_CLOSED &&
1754 old_state != DAPLKA_EP_STATE_DISCONNECTED) {
1755 DERR("ep_free: invalid state %d\n", old_state);
1756 retval = EINVAL;
1757 goto cleanup;
1758 }
1759 ep_rp = NULL;
1760 retval = daplka_hash_remove(&ia_rp->ia_ep_htbl,
1761 args.epf_hkey, (void **)&ep_rp);
1762 if (retval != 0 || ep_rp == NULL) {
1763 /*
1764 * this is only possible if we have two threads
1765 * calling ep_free in parallel.
1766 */
1767 DERR("ep_free: cannot find ep resource\n");
1768 goto cleanup;
1769 }
1770 /* there should not be any outstanding timers */
1771 ASSERT(ep_rp->ep_timer_hkey == 0);
1772
1773 new_state = DAPLKA_EP_STATE_FREED;
1774 daplka_ep_set_state(ep_rp, old_state, new_state);
1775
1776 /* remove reference obtained by lookup */
1777 DAPLKA_RS_UNREF(ep_rp);
1778
1779 /* UNREF calls the actual free function when refcnt is zero */
1780 DAPLKA_RS_UNREF(ep_rp);
1781 return (0);
1782
1783 cleanup:;
1784 daplka_ep_set_state(ep_rp, old_state, new_state);
1785
1786 /* remove reference obtained by lookup */
1787 DAPLKA_RS_UNREF(ep_rp);
1788 return (retval);
1789 }
1790
1791 /*
1792 * The following routines supports the timeout feature of ep_connect.
1793 * Refer to the description of ep_connect for details.
1794 */
1795
1796 /*
1797 * this is the timer processing thread.
1798 */
1799 static void
1800 daplka_timer_thread(void *arg)
1801 {
1802 daplka_timer_info_t *timerp = (daplka_timer_info_t *)arg;
1803 daplka_ep_resource_t *ep_rp;
1804 daplka_evd_event_t *disc_ev = NULL;
1805 ibt_status_t status;
1806 int old_state, new_state;
1807
1808 ep_rp = timerp->ti_ep_res;
1809 ASSERT(ep_rp != NULL);
1810 ASSERT(timerp->ti_tmo_id != 0);
1811 timerp->ti_tmo_id = 0;
1812
1813 new_state = old_state = daplka_ep_get_state(ep_rp);
1814 if (old_state != DAPLKA_EP_STATE_CONNECTING) {
1815 /* unblock hash_ep_free */
1816 mutex_enter(&ep_rp->ep_lock);
1817 ASSERT(ep_rp->ep_timer_hkey != 0);
1818 ep_rp->ep_timer_hkey = 0;
1819 cv_broadcast(&ep_rp->ep_cv);
1820 mutex_exit(&ep_rp->ep_lock);
1821
1822 /* reset state to original state */
1823 daplka_ep_set_state(ep_rp, old_state, new_state);
1824
1825 /* this function will also unref ep_rp */
1826 daplka_timer_info_free(timerp);
1827 return;
1828 }
1829
1830 ASSERT(ep_rp->ep_timer_hkey != 0);
1831 ep_rp->ep_timer_hkey = 0;
1832
1833 /*
1834 * we cannot keep ep_state in TRANSITIONING if we call
1835 * ibt_close_rc_channel in blocking mode. this would cause
1836 * a deadlock because the cm callbacks will be blocked and
1837 * will not be able to wake us up.
1838 */
1839 new_state = DAPLKA_EP_STATE_ABORTING;
1840 daplka_ep_set_state(ep_rp, old_state, new_state);
1841
1842 /*
1843 * when we return from close_rc_channel, all callbacks should have
1844 * completed. we can also be certain that these callbacks did not
1845 * enqueue any events to conn_evd.
1846 */
1847 status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_BLOCKING,
1848 NULL, 0, NULL, NULL, NULL);
1849 if (status != IBT_SUCCESS) {
1850 DERR("timer_thread: ibt_close_rc_channel returned %d\n",
1851 status);
1852 }
1853 old_state = daplka_ep_get_state(ep_rp);
1854
1855 /*
1856 * this is the only thread that can transition ep_state out
1857 * of ABORTING. all other ep operations would fail when
1858 * ep_state is in ABORTING.
1859 */
1860 ASSERT(old_state == DAPLKA_EP_STATE_ABORTING);
1861
1862 disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_SLEEP);
1863 ASSERT(disc_ev != NULL);
1864
1865 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
1866 disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
1867 disc_ev->ee_cmev.ec_cm_is_passive = B_FALSE;
1868 disc_ev->ee_cmev.ec_cm_psep_cookie = 0;
1869 disc_ev->ee_cmev.ec_cm_ev_priv_data = NULL;
1870 disc_ev->ee_cmev.ec_cm_ev_priv_data_len = 0;
1871
1872 D2("timer_thread: enqueue event(%p) evdp(%p)\n",
1873 disc_ev, ep_rp->ep_conn_evd);
1874
1875 new_state = DAPLKA_EP_STATE_DISCONNECTED;
1876 daplka_ep_set_state(ep_rp, old_state, new_state);
1877
1878 daplka_evd_wakeup(ep_rp->ep_conn_evd,
1879 &ep_rp->ep_conn_evd->evd_conn_events, disc_ev);
1880
1881 /* this function will also unref ep_rp */
1882 daplka_timer_info_free(timerp);
1883 }
1884
1885 /*
1886 * dispatches a thread to continue with timer processing.
1887 */
1888 static void
1889 daplka_timer_dispatch(void *arg)
1890 {
1891 /*
1892 * keep rescheduling this function until
1893 * taskq_dispatch succeeds.
1894 */
1895 if (taskq_dispatch(daplka_taskq,
1896 daplka_timer_thread, arg, TQ_NOSLEEP) == 0) {
1897 DERR("timer_dispatch: taskq_dispatch failed, retrying...\n");
1898 (void) timeout(daplka_timer_dispatch, arg, 10);
1899 }
1900 }
1901
1902 /*
1903 * this function is called by the kernel's callout thread.
1904 * we first attempt to remove the timer object from the
1905 * global timer table. if it is found, we dispatch a thread
1906 * to continue processing the timer object. if it is not
1907 * found, that means the timer has been cancelled by someone
1908 * else.
1909 */
1910 static void
1911 daplka_timer_handler(void *arg)
1912 {
1913 uint64_t timer_hkey = (uintptr_t)arg;
1914 daplka_timer_info_t *timerp = NULL;
1915
1916 D2("timer_handler: timer_hkey 0x%llx\n", (longlong_t)timer_hkey);
1917
1918 (void) daplka_hash_remove(&daplka_timer_info_htbl,
1919 timer_hkey, (void **)&timerp);
1920 if (timerp == NULL) {
1921 D2("timer_handler: timer already cancelled\n");
1922 return;
1923 }
1924 daplka_timer_dispatch((void *)timerp);
1925 }
1926
1927 /*
1928 * allocates a timer_info object.
1929 * a reference to a EP is held by this object. this ensures
1930 * that the EP stays valid when a timer is outstanding.
1931 */
1932 static daplka_timer_info_t *
1933 daplka_timer_info_alloc(daplka_ep_resource_t *ep_rp)
1934 {
1935 daplka_timer_info_t *timerp;
1936
1937 timerp = kmem_zalloc(sizeof (*timerp), daplka_km_flags);
1938 if (timerp == NULL) {
1939 DERR("timer_info_alloc: cannot allocate timer info\n");
1940 return (NULL);
1941 }
1942 timerp->ti_ep_res = ep_rp;
1943 timerp->ti_tmo_id = 0;
1944
1945 return (timerp);
1946 }
1947
1948 /*
1949 * Frees the timer_info object.
1950 * we release the EP reference before freeing the object.
1951 */
1952 static void
1953 daplka_timer_info_free(daplka_timer_info_t *timerp)
1954 {
1955 ASSERT(timerp->ti_ep_res != NULL);
1956 DAPLKA_RS_UNREF(timerp->ti_ep_res);
1957 timerp->ti_ep_res = NULL;
1958 ASSERT(timerp->ti_tmo_id == 0);
1959 kmem_free(timerp, sizeof (*timerp));
1960 }
1961
1962 /*
1963 * cancels the timer set by ep_connect.
1964 * returns -1 if timer handling is in progress
1965 * and 0 otherwise.
1966 */
1967 static int
1968 daplka_cancel_timer(daplka_ep_resource_t *ep_rp)
1969 {
1970 /*
1971 * this function can only be called when ep_state
1972 * is frozen.
1973 */
1974 ASSERT(ep_rp->ep_state == DAPLKA_EP_STATE_TRANSITIONING);
1975 if (ep_rp->ep_timer_hkey != 0) {
1976 daplka_timer_info_t *timerp = NULL;
1977
1978 (void) daplka_hash_remove(&daplka_timer_info_htbl,
1979 ep_rp->ep_timer_hkey, (void **)&timerp);
1980 if (timerp == NULL) {
1981 /*
1982 * this is possible if the timer_handler has
1983 * removed the timerp but the taskq thread has
1984 * not transitioned the ep_state to DISCONNECTED.
1985 * we need to reset the ep_state to allow the
1986 * taskq thread to continue with its work. the
1987 * taskq thread will set the ep_timer_hkey to 0
1988 * so we don't have to do it here.
1989 */
1990 DERR("cancel_timer: timer is being processed\n");
1991 return (-1);
1992 }
1993 /*
1994 * we got the timer object. if the handler fires at
1995 * this point, it will not be able to find the object
1996 * and will return immediately. normally, ti_tmo_id gets
1997 * cleared when the handler fires.
1998 */
1999 ASSERT(timerp->ti_tmo_id != 0);
2000
2001 /*
2002 * note that untimeout can possibly call the handler.
2003 * we are safe because the handler will be a no-op.
2004 */
2005 (void) untimeout(timerp->ti_tmo_id);
2006 timerp->ti_tmo_id = 0;
2007 daplka_timer_info_free(timerp);
2008 ep_rp->ep_timer_hkey = 0;
2009 }
2010 return (0);
2011 }
2012
2013 /*
2014 * this function is called by daplka_hash_destroy for
2015 * freeing timer_info objects
2016 */
2017 static void
2018 daplka_hash_timer_free(void *obj)
2019 {
2020 daplka_timer_info_free((daplka_timer_info_t *)obj);
2021 }
2022
2023 /* ARGSUSED */
2024 static uint16_t
2025 daplka_hellomsg_cksum(DAPL_PRIVATE *dp)
2026 {
2027 uint8_t *bp;
2028 int i;
2029 uint16_t cksum = 0;
2030
2031 bp = (uint8_t *)dp;
2032 for (i = 0; i < sizeof (DAPL_PRIVATE); i++) {
2033 cksum += bp[i];
2034 }
2035 return (cksum);
2036 }
2037
2038 /*
2039 * ep_connect is called by the client to initiate a connection to a
2040 * remote service point. It is a non-blocking call. If a non-zero
2041 * timeout is specified by the client, a timer will be set just before
2042 * returning from ep_connect. Upon a successful return from ep_connect,
2043 * the client will call evd_wait to wait for the connection to complete.
2044 * If the connection is rejected or has failed due to an error, the
2045 * client will be notified with an event containing the appropriate error
2046 * code. If the connection is accepted, the client will be notified with
2047 * the CONN_ESTABLISHED event. If the timer expires before either of the
2048 * above events (error or established), a TIMED_OUT event will be delivered
2049 * to the client.
2050 *
2051 * the complicated part of the timer logic is the handling of race
2052 * conditions with CM callbacks. we need to ensure that either the CM or
2053 * the timer thread gets to deliver an event, but not both. when the
2054 * CM callback is about to deliver an event, it always tries to cancel
2055 * the outstanding timer. if cancel_timer indicates a that the timer is
2056 * already being processed, the CM callback will simply return without
2057 * delivering an event. when the timer thread executes, it tries to check
2058 * if the EP is still in CONNECTING state (timers only work on the active
2059 * side). if the EP is not in this state, the timer thread will return
2060 * without delivering an event.
2061 */
2062 /* ARGSUSED */
2063 static int
2064 daplka_ep_connect(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
2065 cred_t *cred, int *rvalp)
2066 {
2067 daplka_ep_resource_t *ep_rp = NULL;
2068 dapl_ep_connect_t args;
2069 daplka_timer_info_t *timerp = NULL;
2070 uint32_t old_state, new_state;
2071 boolean_t timer_inserted = B_FALSE;
2072 uint64_t timer_hkey = 0;
2073 ibt_path_info_t path_info;
2074 ibt_path_attr_t path_attr;
2075 ibt_hca_attr_t *hca_attrp;
2076 ibt_chan_open_args_t chan_args;
2077 ibt_status_t status = IBT_SUCCESS;
2078 uint8_t num_paths;
2079 void *priv_data;
2080 DAPL_PRIVATE *dp;
2081 int retval = 0;
2082 ib_gid_t *sgid;
2083 ib_gid_t *dgid;
2084 uint64_t dgid_ored;
2085 ibt_ar_t ar_query_s;
2086 ibt_ar_t ar_result_s;
2087 ibt_path_flags_t pathflags;
2088
2089 D3("ep_connect: enter\n");
2090 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_connect_t),
2091 mode);
2092 if (retval != 0) {
2093 DERR("ep_connect: copyin error %d\n", retval);
2094 return (EFAULT);
2095 }
2096 ep_rp = (daplka_ep_resource_t *)
2097 daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epc_hkey);
2098 if (ep_rp == NULL) {
2099 DERR("ep_connect: cannot find ep resource\n");
2100 return (EINVAL);
2101 }
2102 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
2103
2104 new_state = old_state = daplka_ep_get_state(ep_rp);
2105 if (old_state != DAPLKA_EP_STATE_CLOSED) {
2106 DERR("ep_connect: invalid state %d\n", old_state);
2107 retval = EINVAL;
2108 goto cleanup;
2109 }
2110 if (args.epc_priv_sz > DAPL_MAX_PRIVATE_DATA_SIZE) {
2111 DERR("ep_connect: private data len (%d) exceeded "
2112 "max size %d\n", args.epc_priv_sz,
2113 DAPL_MAX_PRIVATE_DATA_SIZE);
2114 retval = EINVAL;
2115 goto cleanup;
2116 }
2117
2118 /*
2119 * check for remote ipaddress to dgid resolution needs ATS
2120 */
2121 dgid = &args.epc_dgid;
2122 dgid_ored = dgid->gid_guid | dgid->gid_prefix;
2123 #if defined(DAPLKA_DEBUG_FORCE_ATS)
2124 dgid_ored = 0ULL;
2125 #endif /* DAPLKA_DEBUG_FORCE_ATS */
2126 /* check for unidentified dgid */
2127 if (dgid_ored == 0ULL) {
2128 /*
2129 * setup for ibt_query_ar()
2130 */
2131 sgid = &ia_rp->ia_hca_sgid;
2132 ar_query_s.ar_gid.gid_guid = 0ULL;
2133 ar_query_s.ar_gid.gid_prefix = 0ULL;
2134 ar_query_s.ar_pkey = 0;
2135 bcopy(args.epc_raddr_sadata.iad_sadata,
2136 ar_query_s.ar_data, DAPL_ATS_NBYTES);
2137 #define UR(b) ar_query_s.ar_data[(b)]
2138 D3("daplka_ep_connect: SA[8] %d.%d.%d.%d\n",
2139 UR(8), UR(9), UR(10), UR(11));
2140 D3("daplka_ep_connect: SA[12] %d.%d.%d.%d\n",
2141 UR(12), UR(13), UR(14), UR(15));
2142 status = ibt_query_ar(sgid, &ar_query_s, &ar_result_s);
2143 if (status != IBT_SUCCESS) {
2144 DERR("ep_connect: ibt_query_ar returned %d\n", status);
2145 *rvalp = (int)status;
2146 retval = 0;
2147 goto cleanup;
2148 }
2149 /*
2150 * dgid identified from SA record
2151 */
2152 dgid = &ar_result_s.ar_gid;
2153 D2("daplka_ep_connect: ATS dgid=%llx:%llx\n",
2154 (longlong_t)dgid->gid_prefix, (longlong_t)dgid->gid_guid);
2155 }
2156
2157 bzero(&path_info, sizeof (ibt_path_info_t));
2158 bzero(&path_attr, sizeof (ibt_path_attr_t));
2159 bzero(&chan_args, sizeof (ibt_chan_open_args_t));
2160
2161 path_attr.pa_dgids = dgid;
2162 path_attr.pa_num_dgids = 1;
2163 /*
2164 * don't set sid in path_attr saves 1 SA query
2165 * Also makes server side not to write the service record
2166 */
2167 path_attr.pa_sgid = ia_rp->ia_hca_sgid;
2168 path_attr.pa_pkey = ia_rp->ia_port_pkey;
2169
2170 /* save the connection ep - struct copy */
2171 ep_rp->ep_sgid = ia_rp->ia_hca_sgid;
2172 ep_rp->ep_dgid = *dgid;
2173
2174 num_paths = 0;
2175 pathflags = IBT_PATH_PKEY;
2176 /* enable APM on remote port but not on loopback case */
2177 if (daplka_apm && ((dgid->gid_prefix != path_attr.pa_sgid.gid_prefix) ||
2178 (dgid->gid_guid != path_attr.pa_sgid.gid_guid))) {
2179 pathflags |= IBT_PATH_APM;
2180 }
2181 status = ibt_get_paths(daplka_dev->daplka_clnt_hdl,
2182 pathflags, &path_attr, 1, &path_info, &num_paths);
2183
2184 if (status != IBT_SUCCESS && status != IBT_INSUFF_DATA) {
2185 DERR("ep_connect: ibt_get_paths returned %d paths %d\n",
2186 status, num_paths);
2187 *rvalp = (int)status;
2188 retval = 0;
2189 goto cleanup;
2190 }
2191 /* fill in the sid directly to path_info */
2192 path_info.pi_sid = args.epc_sid;
2193 hca_attrp = &ia_rp->ia_hca->hca_attr;
2194
2195 /* fill in open channel args */
2196 chan_args.oc_path = &path_info;
2197 chan_args.oc_cm_handler = daplka_cm_rc_handler;
2198 chan_args.oc_cm_clnt_private = (void *)ep_rp;
2199 chan_args.oc_rdma_ra_out = hca_attrp->hca_max_rdma_out_chan;
2200 chan_args.oc_rdma_ra_in = hca_attrp->hca_max_rdma_in_chan;
2201 chan_args.oc_path_retry_cnt = 7; /* 3-bit field */
2202 chan_args.oc_path_rnr_retry_cnt = IBT_RNR_INFINITE_RETRY;
2203
2204 ASSERT(args.epc_priv_sz > 0);
2205 priv_data = (void *)args.epc_priv;
2206
2207 chan_args.oc_priv_data_len = args.epc_priv_sz;
2208 chan_args.oc_priv_data = priv_data;
2209
2210 /*
2211 * calculate checksum value of hello message and
2212 * put hello message in networking byte order
2213 */
2214 dp = (DAPL_PRIVATE *)priv_data;
2215 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*dp))
2216 dp->hello_msg.hi_port = htons(dp->hello_msg.hi_port);
2217 dp->hello_msg.hi_checksum = 0;
2218 dp->hello_msg.hi_checksum = htons(daplka_hellomsg_cksum(dp));
2219 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*dp))
2220
2221 if (args.epc_timeout > 0) {
2222 /*
2223 * increment refcnt before passing reference to
2224 * timer_info_alloc.
2225 */
2226 DAPLKA_RS_REF(ep_rp);
2227 timerp = daplka_timer_info_alloc(ep_rp);
2228 if (timerp == NULL) {
2229 DERR("ep_connect: cannot allocate timer\n");
2230 /*
2231 * we need to remove the reference if
2232 * allocation failed.
2233 */
2234 DAPLKA_RS_UNREF(ep_rp);
2235 retval = ENOMEM;
2236 goto cleanup;
2237 }
2238 /*
2239 * We generate our own hkeys so that timer_hkey can fit
2240 * into a pointer and passed as an arg to timeout()
2241 */
2242 timer_hkey = (uint64_t)daplka_timer_hkey_gen();
2243 retval = daplka_hash_insert(&daplka_timer_info_htbl,
2244 &timer_hkey, (void *)timerp);
2245 if (retval != 0) {
2246 DERR("ep_connect: cannot insert timer info\n");
2247 goto cleanup;
2248 }
2249 ASSERT(ep_rp->ep_timer_hkey == 0);
2250 ep_rp->ep_timer_hkey = timer_hkey;
2251 timer_inserted = B_TRUE;
2252 D2("ep_connect: timer_hkey = 0x%llx\n",
2253 (longlong_t)timer_hkey);
2254 }
2255 status = ibt_open_rc_channel(ep_rp->ep_chan_hdl, IBT_OCHAN_NO_FLAGS,
2256 IBT_NONBLOCKING, &chan_args, NULL);
2257
2258 if (status != IBT_SUCCESS) {
2259 DERR("ep_connect: ibt_open_rc_channel returned %d\n", status);
2260 *rvalp = (int)status;
2261 retval = 0;
2262 goto cleanup;
2263 }
2264 /*
2265 * if a cm callback gets called at this point, it'll have to wait until
2266 * ep_state becomes connecting (or some other state if another thread
2267 * manages to get ahead of the callback). this guarantees that the
2268 * callback will not touch the timer until it gets set.
2269 */
2270 if (timerp != NULL) {
2271 clock_t tmo;
2272
2273 tmo = drv_usectohz((clock_t)args.epc_timeout);
2274 /*
2275 * We generate our own 32 bit timer_hkey so that it can fit
2276 * into a pointer
2277 */
2278 ASSERT(timer_hkey != 0);
2279 timerp->ti_tmo_id = timeout(daplka_timer_handler,
2280 (void *)(uintptr_t)timer_hkey, tmo);
2281 }
2282 new_state = DAPLKA_EP_STATE_CONNECTING;
2283
2284 cleanup:;
2285 if (timerp != NULL && (retval != 0 || status != IBT_SUCCESS)) {
2286 /*
2287 * if ibt_open_rc_channel failed, the timerp must still
2288 * be in daplka_timer_info_htbl because neither the cm
2289 * callback nor the timer_handler will be called.
2290 */
2291 if (timer_inserted) {
2292 daplka_timer_info_t *new_timerp = NULL;
2293
2294 ASSERT(timer_hkey != 0);
2295 (void) daplka_hash_remove(&daplka_timer_info_htbl,
2296 timer_hkey, (void **)&new_timerp);
2297 ASSERT(new_timerp == timerp);
2298 ep_rp->ep_timer_hkey = 0;
2299 }
2300 daplka_timer_info_free(timerp);
2301 }
2302 daplka_ep_set_state(ep_rp, old_state, new_state);
2303 DAPLKA_RS_UNREF(ep_rp);
2304 D3("ep_connect: exit\n");
2305 return (retval);
2306 }
2307
2308 /*
2309 * ep_disconnect closes a connection with a remote peer.
2310 * if a connection has not been established, ep_disconnect
2311 * will instead flush all recv bufs posted to this channel.
2312 * if the EP state is CONNECTED, CONNECTING or ACCEPTING upon
2313 * entry to ep_disconnect, the EP state will transition to
2314 * DISCONNECTING upon exit. the CM callbacks triggered by
2315 * ibt_close_rc_channel will cause EP state to become
2316 * DISCONNECTED. This function is a no-op if EP state is
2317 * DISCONNECTED.
2318 */
2319 /* ARGSUSED */
2320 static int
2321 daplka_ep_disconnect(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
2322 cred_t *cred, int *rvalp)
2323 {
2324 daplka_ep_resource_t *ep_rp = NULL;
2325 dapl_ep_disconnect_t args;
2326 ibt_status_t status;
2327 uint32_t old_state, new_state;
2328 int retval = 0;
2329
2330 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_disconnect_t),
2331 mode);
2332 if (retval != 0) {
2333 DERR("ep_disconnect: copyin error %d\n", retval);
2334 return (EFAULT);
2335 }
2336 ep_rp = (daplka_ep_resource_t *)
2337 daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epd_hkey);
2338 if (ep_rp == NULL) {
2339 DERR("ep_disconnect: cannot find ep resource\n");
2340 return (EINVAL);
2341 }
2342 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
2343
2344 new_state = old_state = daplka_ep_get_state(ep_rp);
2345 if (old_state != DAPLKA_EP_STATE_CONNECTED &&
2346 old_state != DAPLKA_EP_STATE_CONNECTING &&
2347 old_state != DAPLKA_EP_STATE_ACCEPTING &&
2348 old_state != DAPLKA_EP_STATE_DISCONNECTED &&
2349 old_state != DAPLKA_EP_STATE_DISCONNECTING &&
2350 old_state != DAPLKA_EP_STATE_CLOSED) {
2351 DERR("ep_disconnect: invalid state %d\n", old_state);
2352 retval = EINVAL;
2353 goto cleanup;
2354 }
2355
2356 if ((old_state == DAPLKA_EP_STATE_DISCONNECTED) ||
2357 (old_state == DAPLKA_EP_STATE_DISCONNECTING)) {
2358 D2("ep_disconnect: ep already disconnected\n");
2359 retval = 0;
2360 /* we leave the state as DISCONNECTED */
2361 goto cleanup;
2362 }
2363 if (old_state == DAPLKA_EP_STATE_CONNECTING ||
2364 old_state == DAPLKA_EP_STATE_ACCEPTING) {
2365 D2("ep_disconnect: aborting, old_state = %d\n", old_state);
2366 }
2367
2368 /*
2369 * according to the udapl spec, ep_disconnect should
2370 * flush the channel if the channel is not CONNECTED.
2371 */
2372 if (old_state == DAPLKA_EP_STATE_CLOSED) {
2373 status = ibt_flush_channel(ep_rp->ep_chan_hdl);
2374 if (status != IBT_SUCCESS) {
2375 DERR("ep_disconnect: ibt_flush_channel failed %d\n",
2376 status);
2377 *rvalp = (int)status;
2378 }
2379 retval = 0;
2380 /* we leave the state as CLOSED */
2381 goto cleanup;
2382 }
2383
2384 new_state = DAPLKA_EP_STATE_DISCONNECTING;
2385 daplka_ep_set_state(ep_rp, old_state, new_state);
2386 status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_NONBLOCKING,
2387 NULL, 0, NULL, NULL, NULL);
2388
2389 if (status == IBT_SUCCESS) {
2390 DAPLKA_RS_UNREF(ep_rp);
2391 return (retval);
2392 } else {
2393 DERR("ep_disconnect: ibt_close_rc_channel returned %d\n",
2394 status);
2395 *rvalp = (int)status;
2396 retval = 0;
2397 new_state = old_state;
2398 }
2399
2400 cleanup:;
2401 daplka_ep_set_state(ep_rp, old_state, new_state);
2402 DAPLKA_RS_UNREF(ep_rp);
2403 return (retval);
2404 }
2405
2406 /*
2407 * this function resets the EP to a usable state (ie. from
2408 * DISCONNECTED to CLOSED). this function is best implemented using
2409 * the ibt_recycle_channel interface. until that is available, we will
2410 * instead clone and tear down the existing channel and replace the
2411 * existing channel with the cloned one.
2412 */
2413 /* ARGSUSED */
2414 static int
2415 daplka_ep_reinit(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
2416 cred_t *cred, int *rvalp)
2417 {
2418 daplka_ep_resource_t *ep_rp = NULL;
2419 dapl_ep_reinit_t args;
2420 ibt_status_t status;
2421 uint32_t old_state, new_state;
2422 int retval = 0;
2423
2424 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_reinit_t),
2425 mode);
2426 if (retval != 0) {
2427 DERR("reinit: copyin error %d\n", retval);
2428 return (EFAULT);
2429 }
2430 ep_rp = (daplka_ep_resource_t *)
2431 daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epri_hkey);
2432 if (ep_rp == NULL) {
2433 DERR("reinit: cannot find ep resource\n");
2434 return (EINVAL);
2435 }
2436 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
2437 new_state = old_state = daplka_ep_get_state(ep_rp);
2438 if ((old_state != DAPLKA_EP_STATE_CLOSED) &&
2439 (old_state != DAPLKA_EP_STATE_DISCONNECTED)) {
2440 DERR("reinit: invalid state %d\n", old_state);
2441 retval = EINVAL;
2442 goto cleanup;
2443 }
2444
2445 status = ibt_recycle_rc(ep_rp->ep_chan_hdl,
2446 IBT_CEP_RDMA_RD|IBT_CEP_RDMA_WR,
2447 ia_rp->ia_port_num, NULL, NULL);
2448 if (status != IBT_SUCCESS) {
2449 DERR("reinit: unable to clone channel\n");
2450 *rvalp = (int)status;
2451 retval = 0;
2452 goto cleanup;
2453 }
2454 new_state = DAPLKA_EP_STATE_CLOSED;
2455
2456 cleanup:;
2457 daplka_ep_set_state(ep_rp, old_state, new_state);
2458 DAPLKA_RS_UNREF(ep_rp);
2459 return (retval);
2460 }
2461
2462 /*
2463 * destroys a EP resource.
2464 * called when refcnt drops to zero.
2465 */
2466 static int
2467 daplka_ep_destroy(daplka_resource_t *gen_rp)
2468 {
2469 daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)gen_rp;
2470 ibt_status_t status;
2471
2472 ASSERT(DAPLKA_RS_REFCNT(ep_rp) == 0);
2473 ASSERT(ep_rp->ep_state == DAPLKA_EP_STATE_FREED);
2474
2475 /*
2476 * by the time we get here, we can be sure that
2477 * there is no outstanding timer.
2478 */
2479 ASSERT(ep_rp->ep_timer_hkey == 0);
2480
2481 D3("ep_destroy: entering, ep_rp 0x%p, rnum %d\n",
2482 ep_rp, DAPLKA_RS_RNUM(ep_rp));
2483 /*
2484 * free rc channel
2485 */
2486 if (ep_rp->ep_chan_hdl != NULL) {
2487 mutex_enter(&daplka_dev->daplka_mutex);
2488 ibt_set_chan_private(ep_rp->ep_chan_hdl, NULL);
2489 mutex_exit(&daplka_dev->daplka_mutex);
2490 status = daplka_ibt_free_channel(ep_rp, ep_rp->ep_chan_hdl);
2491 if (status != IBT_SUCCESS) {
2492 DERR("ep_free: ibt_free_channel returned %d\n",
2493 status);
2494 }
2495 ep_rp->ep_chan_hdl = NULL;
2496 D3("ep_destroy: qp freed, rnum %d\n", DAPLKA_RS_RNUM(ep_rp));
2497 }
2498 /*
2499 * release all references
2500 */
2501 if (ep_rp->ep_snd_evd != NULL) {
2502 DAPLKA_RS_UNREF(ep_rp->ep_snd_evd);
2503 ep_rp->ep_snd_evd = NULL;
2504 }
2505 if (ep_rp->ep_rcv_evd != NULL) {
2506 DAPLKA_RS_UNREF(ep_rp->ep_rcv_evd);
2507 ep_rp->ep_rcv_evd = NULL;
2508 }
2509 if (ep_rp->ep_conn_evd != NULL) {
2510 DAPLKA_RS_UNREF(ep_rp->ep_conn_evd);
2511 ep_rp->ep_conn_evd = NULL;
2512 }
2513 if (ep_rp->ep_srq_res != NULL) {
2514 DAPLKA_RS_UNREF(ep_rp->ep_srq_res);
2515 ep_rp->ep_srq_res = NULL;
2516 }
2517 if (ep_rp->ep_pd_res != NULL) {
2518 DAPLKA_RS_UNREF(ep_rp->ep_pd_res);
2519 ep_rp->ep_pd_res = NULL;
2520 }
2521 cv_destroy(&ep_rp->ep_cv);
2522 mutex_destroy(&ep_rp->ep_lock);
2523
2524 DAPLKA_RS_FINI(ep_rp);
2525 kmem_free(ep_rp, sizeof (daplka_ep_resource_t));
2526 D3("ep_destroy: exiting, ep_rp 0x%p\n", ep_rp);
2527 return (0);
2528 }
2529
2530 /*
2531 * this function is called by daplka_hash_destroy for
2532 * freeing EP resource objects
2533 */
2534 static void
2535 daplka_hash_ep_free(void *obj)
2536 {
2537 daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)obj;
2538 ibt_status_t status;
2539 uint32_t old_state, new_state;
2540 int retval;
2541
2542 old_state = daplka_ep_get_state(ep_rp);
2543 retval = daplka_cancel_timer(ep_rp);
2544 new_state = DAPLKA_EP_STATE_FREED;
2545 daplka_ep_set_state(ep_rp, old_state, new_state);
2546
2547 if (retval != 0) {
2548 D2("hash_ep_free: ep_rp 0x%p "
2549 "timer is still being processed\n", ep_rp);
2550 mutex_enter(&ep_rp->ep_lock);
2551 if (ep_rp->ep_timer_hkey != 0) {
2552 D2("hash_ep_free: ep_rp 0x%p "
2553 "waiting for timer_hkey to be 0\n", ep_rp);
2554 cv_wait(&ep_rp->ep_cv, &ep_rp->ep_lock);
2555 }
2556 mutex_exit(&ep_rp->ep_lock);
2557 }
2558
2559 /* call ibt_close_rc_channel regardless of what state we are in */
2560 status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_BLOCKING,
2561 NULL, 0, NULL, NULL, NULL);
2562 if (status != IBT_SUCCESS) {
2563 if (old_state == DAPLKA_EP_STATE_CONNECTED ||
2564 old_state == DAPLKA_EP_STATE_CONNECTING ||
2565 old_state == DAPLKA_EP_STATE_ACCEPTING) {
2566 DERR("hash_ep_free: ep_rp 0x%p state %d "
2567 "unexpected error %d from close_rc_channel\n",
2568 ep_rp, old_state, status);
2569 }
2570 D2("hash_ep_free: close_rc_channel, status %d\n", status);
2571 }
2572
2573 DAPLKA_RS_UNREF(ep_rp);
2574 }
2575
2576 /*
2577 * creates a EVD resource.
2578 * a EVD is used by the client to wait for events from one
2579 * or more sources.
2580 */
2581 /* ARGSUSED */
2582 static int
2583 daplka_evd_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
2584 cred_t *cred, int *rvalp)
2585 {
2586 daplka_evd_resource_t *evd_rp = NULL;
2587 daplka_async_evd_hkey_t *async_evd;
2588 ibt_hca_attr_t *hca_attrp;
2589 ibt_cq_attr_t cq_attr;
2590 dapl_evd_create_t args;
2591 uint64_t evd_hkey = 0;
2592 boolean_t inserted = B_FALSE;
2593 int retval = 0;
2594 ibt_status_t status;
2595
2596 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_create_t),
2597 mode);
2598 if (retval != 0) {
2599 DERR("evd_create: copyin error %d", retval);
2600 return (EFAULT);
2601 }
2602 if ((args.evd_flags &
2603 ~(DAT_EVD_DEFAULT_FLAG | DAT_EVD_SOFTWARE_FLAG)) != 0) {
2604 DERR("evd_create: invalid flags 0x%x\n", args.evd_flags);
2605 return (EINVAL);
2606 }
2607
2608 evd_rp = kmem_zalloc(sizeof (daplka_evd_resource_t), daplka_km_flags);
2609 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*evd_rp))
2610 DAPLKA_RS_INIT(evd_rp, DAPL_TYPE_EVD,
2611 DAPLKA_RS_RNUM(ia_rp), daplka_evd_destroy);
2612
2613 mutex_init(&evd_rp->evd_lock, NULL, MUTEX_DRIVER, NULL);
2614 cv_init(&evd_rp->evd_cv, NULL, CV_DRIVER, NULL);
2615 evd_rp->evd_hca = ia_rp->ia_hca;
2616 evd_rp->evd_flags = args.evd_flags;
2617 evd_rp->evd_hca_hdl = ia_rp->ia_hca_hdl;
2618 evd_rp->evd_cookie = args.evd_cookie;
2619 evd_rp->evd_cno_res = NULL;
2620 evd_rp->evd_cr_events.eel_event_type = DAPLKA_EVD_CM_EVENTS;
2621 evd_rp->evd_conn_events.eel_event_type = DAPLKA_EVD_CM_EVENTS;
2622 evd_rp->evd_async_events.eel_event_type = DAPLKA_EVD_ASYNC_EVENTS;
2623
2624 /*
2625 * if the client specified a non-zero cno_hkey, we
2626 * lookup the cno and save the reference for later use.
2627 */
2628 if (args.evd_cno_hkey > 0) {
2629 daplka_cno_resource_t *cno_rp;
2630
2631 cno_rp = (daplka_cno_resource_t *)
2632 daplka_hash_lookup(&ia_rp->ia_cno_htbl,
2633 args.evd_cno_hkey);
2634 if (cno_rp == NULL) {
2635 DERR("evd_create: cannot find cno resource\n");
2636 goto cleanup;
2637 }
2638 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
2639 evd_rp->evd_cno_res = cno_rp;
2640 }
2641 hca_attrp = &ia_rp->ia_hca->hca_attr;
2642 if ((evd_rp->evd_flags &
2643 (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) != 0) {
2644 if (args.evd_cq_size > hca_attrp->hca_max_cq_sz) {
2645 DERR("evd_create: invalid cq size %d",
2646 args.evd_cq_size);
2647 retval = EINVAL;
2648 goto cleanup;
2649 }
2650 cq_attr.cq_size = args.evd_cq_size;
2651 cq_attr.cq_sched = NULL;
2652 cq_attr.cq_flags = IBT_CQ_USER_MAP;
2653
2654 status = daplka_ibt_alloc_cq(evd_rp, evd_rp->evd_hca_hdl,
2655 &cq_attr, &evd_rp->evd_cq_hdl, &evd_rp->evd_cq_real_size);
2656
2657 if (status != IBT_SUCCESS) {
2658 DERR("evd_create: ibt_alloc_cq returned %d", status);
2659 *rvalp = (int)status;
2660 retval = 0;
2661 goto cleanup;
2662 }
2663
2664 /*
2665 * store evd ptr with cq_hdl
2666 * mutex is only needed for race of "destroy" and "async"
2667 */
2668 mutex_enter(&daplka_dev->daplka_mutex);
2669 ibt_set_cq_private(evd_rp->evd_cq_hdl, (void *)evd_rp);
2670 mutex_exit(&daplka_dev->daplka_mutex);
2671
2672 /* Get HCA-specific data_out info */
2673 status = ibt_ci_data_out(evd_rp->evd_hca_hdl,
2674 IBT_CI_NO_FLAGS, IBT_HDL_CQ, (void *)evd_rp->evd_cq_hdl,
2675 &args.evd_cq_data_out, sizeof (args.evd_cq_data_out));
2676
2677 if (status != IBT_SUCCESS) {
2678 DERR("evd_create: ibt_ci_data_out error(%d)", status);
2679 *rvalp = (int)status;
2680 retval = 0;
2681 goto cleanup;
2682 }
2683
2684 args.evd_cq_real_size = evd_rp->evd_cq_real_size;
2685
2686 ibt_set_cq_handler(evd_rp->evd_cq_hdl, daplka_cq_handler,
2687 (void *)evd_rp);
2688 }
2689
2690 retval = daplka_hash_insert(&ia_rp->ia_evd_htbl,
2691 &evd_hkey, (void *)evd_rp);
2692 if (retval != 0) {
2693 DERR("evd_ceate: cannot insert evd %d\n", retval);
2694 goto cleanup;
2695 }
2696 inserted = B_TRUE;
2697 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*evd_rp))
2698
2699 /*
2700 * If this evd handles async events need to add to the IA resource
2701 * async evd list
2702 */
2703 if (evd_rp->evd_flags & DAT_EVD_ASYNC_FLAG) {
2704 async_evd = kmem_zalloc(sizeof (daplka_async_evd_hkey_t),
2705 daplka_km_flags);
2706 /* add the evd to the head of the list */
2707 mutex_enter(&ia_rp->ia_lock);
2708 async_evd->aeh_evd_hkey = evd_hkey;
2709 async_evd->aeh_next = ia_rp->ia_async_evd_hkeys;
2710 ia_rp->ia_async_evd_hkeys = async_evd;
2711 mutex_exit(&ia_rp->ia_lock);
2712 }
2713
2714 args.evd_hkey = evd_hkey;
2715 retval = copyout(&args, (void *)arg, sizeof (dapl_evd_create_t));
2716 if (retval != 0) {
2717 DERR("evd_create: copyout error %d\n", retval);
2718 retval = EFAULT;
2719 goto cleanup;
2720 }
2721 return (0);
2722
2723 cleanup:;
2724 if (inserted) {
2725 daplka_evd_resource_t *free_rp = NULL;
2726
2727 (void) daplka_hash_remove(&ia_rp->ia_evd_htbl, evd_hkey,
2728 (void **)&free_rp);
2729 if (free_rp != evd_rp) {
2730 DERR("evd_create: cannot remove evd\n");
2731 /*
2732 * we can only get here if another thread
2733 * has completed the cleanup in evd_free
2734 */
2735 return (retval);
2736 }
2737 }
2738 DAPLKA_RS_UNREF(evd_rp);
2739 return (retval);
2740 }
2741
2742 /*
2743 * resizes CQ and returns new mapping info to library.
2744 */
2745 /* ARGSUSED */
2746 static int
2747 daplka_cq_resize(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
2748 cred_t *cred, int *rvalp)
2749 {
2750 daplka_evd_resource_t *evd_rp = NULL;
2751 ibt_hca_attr_t *hca_attrp;
2752 dapl_cq_resize_t args;
2753 ibt_status_t status;
2754 int retval = 0;
2755
2756 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cq_resize_t),
2757 mode);
2758 if (retval != 0) {
2759 DERR("cq_resize: copyin error %d\n", retval);
2760 return (EFAULT);
2761 }
2762
2763 /* get evd resource */
2764 evd_rp = (daplka_evd_resource_t *)
2765 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.cqr_evd_hkey);
2766 if (evd_rp == NULL) {
2767 DERR("cq_resize: cannot find evd resource\n");
2768 return (EINVAL);
2769 }
2770 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
2771
2772 hca_attrp = &ia_rp->ia_hca->hca_attr;
2773 if (args.cqr_cq_new_size > hca_attrp->hca_max_cq_sz) {
2774 DERR("cq_resize: invalid cq size %d", args.cqr_cq_new_size);
2775 retval = EINVAL;
2776 goto cleanup;
2777 }
2778 /*
2779 * If ibt_resize_cq fails that it is primarily due to resource
2780 * shortage. Per IB spec resize will never loose events and
2781 * a resize error leaves the CQ intact. Therefore even if the
2782 * resize request fails we proceed and get the mapping data
2783 * from the CQ so that the library can mmap it.
2784 */
2785 status = ibt_resize_cq(evd_rp->evd_cq_hdl, args.cqr_cq_new_size,
2786 &args.cqr_cq_real_size);
2787 if (status != IBT_SUCCESS) {
2788 /* we return the size of the old CQ if resize fails */
2789 args.cqr_cq_real_size = evd_rp->evd_cq_real_size;
2790 ASSERT(status != IBT_CQ_HDL_INVALID);
2791 DERR("cq_resize: ibt_resize_cq failed:%d\n", status);
2792 } else {
2793 mutex_enter(&evd_rp->evd_lock);
2794 evd_rp->evd_cq_real_size = args.cqr_cq_real_size;
2795 mutex_exit(&evd_rp->evd_lock);
2796 }
2797
2798 D2("cq_resize(%d): done new_sz(%u) real_sz(%u)\n",
2799 DAPLKA_RS_RNUM(evd_rp),
2800 args.cqr_cq_new_size, args.cqr_cq_real_size);
2801
2802 /* Get HCA-specific data_out info */
2803 status = ibt_ci_data_out(evd_rp->evd_hca_hdl,
2804 IBT_CI_NO_FLAGS, IBT_HDL_CQ, (void *)evd_rp->evd_cq_hdl,
2805 &args.cqr_cq_data_out, sizeof (args.cqr_cq_data_out));
2806 if (status != IBT_SUCCESS) {
2807 DERR("cq_resize: ibt_ci_data_out error(%d)\n", status);
2808 /* return ibt_ci_data_out status */
2809 *rvalp = (int)status;
2810 retval = 0;
2811 goto cleanup;
2812 }
2813
2814 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_cq_resize_t),
2815 mode);
2816 if (retval != 0) {
2817 DERR("cq_resize: copyout error %d\n", retval);
2818 retval = EFAULT;
2819 goto cleanup;
2820 }
2821
2822 cleanup:;
2823 if (evd_rp != NULL) {
2824 DAPLKA_RS_UNREF(evd_rp);
2825 }
2826 return (retval);
2827 }
2828
2829 /*
2830 * Routine to copyin the event poll message so that 32 bit libraries
2831 * can be safely supported
2832 */
2833 int
2834 daplka_event_poll_copyin(intptr_t inarg, dapl_event_poll_t *outarg, int mode)
2835 {
2836 int retval;
2837
2838 #ifdef _MULTI_DATAMODEL
2839 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
2840 dapl_event_poll32_t args32;
2841
2842 retval = ddi_copyin((void *)inarg, &args32,
2843 sizeof (dapl_event_poll32_t), mode);
2844 if (retval != 0) {
2845 DERR("event_poll_copyin: 32bit error %d\n", retval);
2846 return (EFAULT);
2847 }
2848
2849 outarg->evp_evd_hkey = args32.evp_evd_hkey;
2850 outarg->evp_threshold = args32.evp_threshold;
2851 outarg->evp_timeout = args32.evp_timeout;
2852 outarg->evp_ep = (dapl_ib_event_t *)(uintptr_t)args32.evp_ep;
2853 outarg->evp_num_ev = args32.evp_num_ev;
2854 outarg->evp_num_polled = args32.evp_num_polled;
2855 return (0);
2856 }
2857 #endif
2858 retval = ddi_copyin((void *)inarg, outarg, sizeof (dapl_event_poll_t),
2859 mode);
2860 if (retval != 0) {
2861 DERR("event_poll: copyin error %d\n", retval);
2862 return (EFAULT);
2863 }
2864
2865 return (0);
2866 }
2867
2868 /*
2869 * Routine to copyout the event poll message so that 32 bit libraries
2870 * can be safely supported
2871 */
2872 int
2873 daplka_event_poll_copyout(dapl_event_poll_t *inarg, intptr_t outarg, int mode)
2874 {
2875 int retval;
2876
2877 #ifdef _MULTI_DATAMODEL
2878 if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
2879 dapl_event_poll32_t args32;
2880
2881 args32.evp_evd_hkey = inarg->evp_evd_hkey;
2882 args32.evp_threshold = inarg->evp_threshold;
2883 args32.evp_timeout = inarg->evp_timeout;
2884 args32.evp_ep = (caddr32_t)(uintptr_t)inarg->evp_ep;
2885 args32.evp_num_ev = inarg->evp_num_ev;
2886 args32.evp_num_polled = inarg->evp_num_polled;
2887
2888 retval = ddi_copyout((void *)&args32, (void *)outarg,
2889 sizeof (dapl_event_poll32_t), mode);
2890 if (retval != 0) {
2891 DERR("event_poll_copyout: 32bit error %d\n", retval);
2892 return (EFAULT);
2893 }
2894 return (0);
2895 }
2896 #endif
2897 retval = ddi_copyout((void *)inarg, (void *)outarg,
2898 sizeof (dapl_event_poll_t), mode);
2899 if (retval != 0) {
2900 DERR("event_poll_copyout: error %d\n", retval);
2901 return (EFAULT);
2902 }
2903
2904 return (0);
2905 }
2906
2907 /*
2908 * fucntion to handle CM REQ RCV private data from Solaris or third parties
2909 */
2910 /* ARGSUSED */
2911 static void
2912 daplka_crevent_privdata_post(daplka_ia_resource_t *ia_rp,
2913 dapl_ib_event_t *evd_rp, daplka_evd_event_t *cr_ev)
2914 {
2915 DAPL_PRIVATE *dp;
2916 ib_gid_t *lgid;
2917 ibt_ar_t ar_query_s;
2918 ibt_ar_t ar_result_s;
2919 DAPL_HELLO_MSG *hip;
2920 uint32_t ipaddr_ord;
2921 ibt_priv_data_len_t clen;
2922 ibt_priv_data_len_t olen;
2923 ibt_status_t status;
2924 uint16_t cksum;
2925
2926 /*
2927 * get private data and len
2928 */
2929 dp = (DAPL_PRIVATE *)cr_ev->ee_cmev.ec_cm_ev_priv_data;
2930 clen = cr_ev->ee_cmev.ec_cm_ev_priv_data_len;
2931 #if defined(DAPLKA_DEBUG_FORCE_ATS)
2932 /* skip the DAPL_PRIVATE chekcsum check */
2933 #else
2934 /* for remote connects */
2935 /* look up hello message in the CM private data area */
2936 if (clen >= sizeof (DAPL_PRIVATE) &&
2937 (dp->hello_msg.hi_vers == DAPL_HELLO_MSG_VERS)) {
2938 cksum = ntohs(dp->hello_msg.hi_checksum);
2939 dp->hello_msg.hi_checksum = 0;
2940 if (daplka_hellomsg_cksum(dp) == cksum) {
2941 D2("daplka_crevent_privdata_post: Solaris msg\n");
2942 evd_rp->ibe_ce.ibce_priv_data_size = clen;
2943 dp->hello_msg.hi_checksum = DAPL_CHECKSUM;
2944 dp->hello_msg.hi_port = ntohs(dp->hello_msg.hi_port);
2945 bcopy(dp, evd_rp->ibe_ce.ibce_priv_data_ptr, clen);
2946 kmem_free(dp, clen);
2947 return;
2948 }
2949 }
2950 #endif /* DAPLKA_DEBUG_FORCE_ATS */
2951
2952 D2("daplka_crevent_privdata_post: 3rd party msg\n");
2953 /* transpose CM private data into hello message */
2954 if (clen) {
2955 olen = clen;
2956 if (clen > DAPL_CONSUMER_MAX_PRIVATE_DATA_SIZE) {
2957 clen = DAPL_CONSUMER_MAX_PRIVATE_DATA_SIZE;
2958 }
2959 bcopy(dp, evd_rp->ibe_ce.ibce_priv_data_ptr, clen);
2960 kmem_free(dp, olen);
2961 } else {
2962 bzero(evd_rp->ibe_ce.ibce_priv_data_ptr,
2963 DAPL_CONSUMER_MAX_PRIVATE_DATA_SIZE);
2964 }
2965 evd_rp->ibe_ce.ibce_priv_data_size = sizeof (DAPL_PRIVATE);
2966 dp = (DAPL_PRIVATE *)evd_rp->ibe_ce.ibce_priv_data_ptr;
2967 /*
2968 * fill in hello message
2969 */
2970 hip = &dp->hello_msg;
2971 hip->hi_checksum = DAPL_CHECKSUM;
2972 hip->hi_clen = clen;
2973 hip->hi_mid = 0;
2974 hip->hi_vers = DAPL_HELLO_MSG_VERS;
2975 hip->hi_port = 0;
2976
2977 /* assign sgid and dgid */
2978 lgid = &ia_rp->ia_hca_sgid;
2979 ar_query_s.ar_gid.gid_prefix =
2980 cr_ev->ee_cmev.ec_cm_req_prim_addr.gid_prefix;
2981 ar_query_s.ar_gid.gid_guid =
2982 cr_ev->ee_cmev.ec_cm_req_prim_addr.gid_guid;
2983 ar_query_s.ar_pkey = ia_rp->ia_port_pkey;
2984 bzero(ar_query_s.ar_data, DAPL_ATS_NBYTES);
2985
2986 /* reverse ip address lookup through ATS */
2987 status = ibt_query_ar(lgid, &ar_query_s, &ar_result_s);
2988 if (status == IBT_SUCCESS) {
2989 bcopy(ar_result_s.ar_data, hip->hi_saaddr, DAPL_ATS_NBYTES);
2990 /* determine the address families */
2991 ipaddr_ord = hip->hi_v4pad[0] | hip->hi_v4pad[1] |
2992 hip->hi_v4pad[2];
2993 if (ipaddr_ord == 0) {
2994 hip->hi_ipv = AF_INET;
2995 } else {
2996 hip->hi_ipv = AF_INET6;
2997 }
2998
2999 #define UL(b) ar_result_s.ar_data[(b)]
3000 D3("daplka_privdata_post: family=%d :SA[8] %d.%d.%d.%d\n",
3001 hip->hi_ipv, UL(8), UL(9), UL(10), UL(11));
3002 D3("daplka_privdata_post: SA[12] %d.%d.%d.%d\n",
3003 UL(12), UL(13), UL(14), UL(15));
3004 } else {
3005 /* non-conformed third parties */
3006 hip->hi_ipv = AF_UNSPEC;
3007 bzero(hip->hi_saaddr, DAPL_ATS_NBYTES);
3008 }
3009 }
3010
3011 /*
3012 * this function is called by evd_wait and evd_dequeue to wait for
3013 * connection events and CQ notifications. typically this function
3014 * is called when the userland CQ is empty and the client has
3015 * specified a non-zero timeout to evd_wait. if the client is
3016 * interested in CQ events, the CQ must be armed in userland prior
3017 * to calling this function.
3018 */
3019 /* ARGSUSED */
3020 static int
3021 daplka_event_poll(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3022 cred_t *cred, int *rvalp)
3023 {
3024 daplka_evd_resource_t *evd_rp = NULL;
3025 dapl_event_poll_t args;
3026 daplka_evd_event_t *head;
3027 dapl_ib_event_t evp_arr[NUM_EVENTS_PER_POLL];
3028 dapl_ib_event_t *evp;
3029 dapl_ib_event_t *evp_start;
3030 size_t evp_size;
3031 int threshold;
3032 clock_t timeout;
3033 uint32_t max_events;
3034 uint32_t num_events = 0;
3035 void *pd;
3036 ibt_priv_data_len_t n;
3037 int retval = 0;
3038 int rc;
3039
3040 retval = daplka_event_poll_copyin(arg, &args, mode);
3041 if (retval != 0) {
3042 return (EFAULT);
3043 }
3044
3045 if ((args.evp_num_ev > 0) && (args.evp_ep == NULL)) {
3046 DERR("event_poll: evp_ep cannot be NULL if num_wc=%d",
3047 args.evp_num_ev);
3048 return (EINVAL);
3049 }
3050 /*
3051 * Note: dequeue requests have a threshold = 0, timeout = 0
3052 */
3053 threshold = args.evp_threshold;
3054
3055 max_events = args.evp_num_ev;
3056 /* ensure library is passing sensible values */
3057 if (max_events < threshold) {
3058 DERR("event_poll: max_events(%d) < threshold(%d)\n",
3059 max_events, threshold);
3060 return (EINVAL);
3061 }
3062 /* Do a sanity check to avoid excessive memory allocation */
3063 if (max_events > DAPL_EVD_MAX_EVENTS) {
3064 DERR("event_poll: max_events(%d) > %d",
3065 max_events, DAPL_EVD_MAX_EVENTS);
3066 return (EINVAL);
3067 }
3068 D4("event_poll: threshold(%d) timeout(0x%llx) max_events(%d)\n",
3069 threshold, (longlong_t)args.evp_timeout, max_events);
3070
3071 /* get evd resource */
3072 evd_rp = (daplka_evd_resource_t *)
3073 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evp_evd_hkey);
3074 if (evd_rp == NULL) {
3075 DERR("event_poll: cannot find evd resource\n");
3076 return (EINVAL);
3077 }
3078 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
3079
3080 /*
3081 * Use event array on the stack if possible
3082 */
3083 if (max_events <= NUM_EVENTS_PER_POLL) {
3084 evp_start = evp = &evp_arr[0];
3085 } else {
3086 evp_size = max_events * sizeof (dapl_ib_event_t);
3087 evp_start = evp = kmem_zalloc(evp_size, daplka_km_flags);
3088 if (evp == NULL) {
3089 DERR("event_poll: kmem_zalloc failed, evp_size %d",
3090 evp_size);
3091 retval = ENOMEM;
3092 goto cleanup;
3093 }
3094 }
3095
3096 /*
3097 * The Event poll algorithm is as follows -
3098 * The library passes a buffer big enough to hold "max_events"
3099 * events. max_events is >= threshold. If at any stage we get
3100 * max_events no. of events we bail. The events are polled in
3101 * the following order -
3102 * 1) Check for CR events in the evd_cr_events list
3103 * 2) Check for Connection events in the evd_connection_events list
3104 *
3105 * If after the above 2 steps we don't have enough(>= threshold) events
3106 * we block for CQ notification and sleep. Upon being woken up we start
3107 * at step 1 again.
3108 */
3109
3110 /*
3111 * Note: this could be 0 or INFINITE or anyother value in microsec
3112 */
3113 if (args.evp_timeout > 0) {
3114 if (args.evp_timeout >= LONG_MAX) {
3115 timeout = LONG_MAX;
3116 } else {
3117 clock_t curr_time = ddi_get_lbolt();
3118
3119 timeout = curr_time +
3120 drv_usectohz((clock_t)args.evp_timeout);
3121 /*
3122 * use the max value if we wrapped around
3123 */
3124 if (timeout <= curr_time) {
3125 timeout = LONG_MAX;
3126 }
3127 }
3128 } else {
3129 timeout = 0;
3130 }
3131
3132 mutex_enter(&evd_rp->evd_lock);
3133 for (;;) {
3134 /*
3135 * If this evd is waiting for CM events check that now.
3136 */
3137 if ((evd_rp->evd_flags & DAT_EVD_CR_FLAG) &&
3138 (evd_rp->evd_cr_events.eel_num_elements > 0)) {
3139 /* dequeue events from evd_cr_events list */
3140 while (head = daplka_evd_event_dequeue(
3141 &evd_rp->evd_cr_events)) {
3142 /*
3143 * populate the evp array
3144 */
3145 evp[num_events].ibe_ev_family = DAPL_CR_EVENTS;
3146 evp[num_events].ibe_ce.ibce_event =
3147 head->ee_cmev.ec_cm_ev_type;
3148 evp[num_events].ibe_ce.ibce_cookie =
3149 (uint64_t)head->ee_cmev.ec_cm_cookie;
3150 evp[num_events].ibe_ce.ibce_psep_cookie =
3151 head->ee_cmev.ec_cm_psep_cookie;
3152 daplka_crevent_privdata_post(ia_rp,
3153 &evp[num_events], head);
3154 kmem_free(head, sizeof (daplka_evd_event_t));
3155
3156 if (++num_events == max_events) {
3157 mutex_exit(&evd_rp->evd_lock);
3158 goto maxevent_reached;
3159 }
3160 }
3161 }
3162
3163 if ((evd_rp->evd_flags & DAT_EVD_CONNECTION_FLAG) &&
3164 (evd_rp->evd_conn_events.eel_num_elements > 0)) {
3165 /* dequeue events from evd_connection_events list */
3166 while ((head = daplka_evd_event_dequeue
3167 (&evd_rp->evd_conn_events))) {
3168 /*
3169 * populate the evp array -
3170 *
3171 */
3172 if (head->ee_cmev.ec_cm_is_passive) {
3173 evp[num_events].ibe_ev_family =
3174 DAPL_PASSIVE_CONNECTION_EVENTS;
3175 } else {
3176 evp[num_events].ibe_ev_family =
3177 DAPL_ACTIVE_CONNECTION_EVENTS;
3178 }
3179 evp[num_events].ibe_ce.ibce_event =
3180 head->ee_cmev.ec_cm_ev_type;
3181 evp[num_events].ibe_ce.ibce_cookie =
3182 (uint64_t)head->ee_cmev.ec_cm_cookie;
3183 evp[num_events].ibe_ce.ibce_psep_cookie =
3184 head->ee_cmev.ec_cm_psep_cookie;
3185
3186 if (head->ee_cmev.ec_cm_ev_priv_data_len > 0) {
3187 pd = head->ee_cmev.ec_cm_ev_priv_data;
3188 n = head->
3189 ee_cmev.ec_cm_ev_priv_data_len;
3190 bcopy(pd, (void *)evp[num_events].
3191 ibe_ce.ibce_priv_data_ptr, n);
3192 evp[num_events].ibe_ce.
3193 ibce_priv_data_size = n;
3194 kmem_free(pd, n);
3195 }
3196
3197 kmem_free(head, sizeof (daplka_evd_event_t));
3198
3199 if (++num_events == max_events) {
3200 mutex_exit(&evd_rp->evd_lock);
3201 goto maxevent_reached;
3202 }
3203 }
3204 }
3205
3206 if ((evd_rp->evd_flags & DAT_EVD_ASYNC_FLAG) &&
3207 (evd_rp->evd_async_events.eel_num_elements > 0)) {
3208 /* dequeue events from evd_async_events list */
3209 while (head = daplka_evd_event_dequeue(
3210 &evd_rp->evd_async_events)) {
3211 /*
3212 * populate the evp array
3213 */
3214 evp[num_events].ibe_ev_family =
3215 DAPL_ASYNC_EVENTS;
3216 evp[num_events].ibe_async.ibae_type =
3217 head->ee_aev.ibae_type;
3218 evp[num_events].ibe_async.ibae_hca_guid =
3219 head->ee_aev.ibae_hca_guid;
3220 evp[num_events].ibe_async.ibae_cookie =
3221 head->ee_aev.ibae_cookie;
3222 evp[num_events].ibe_async.ibae_port =
3223 head->ee_aev.ibae_port;
3224
3225 kmem_free(head, sizeof (daplka_evd_event_t));
3226
3227 if (++num_events == max_events) {
3228 break;
3229 }
3230 }
3231 }
3232
3233 /*
3234 * We have sufficient events for this call so no need to wait
3235 */
3236 if ((threshold > 0) && (num_events >= threshold)) {
3237 mutex_exit(&evd_rp->evd_lock);
3238 break;
3239 }
3240
3241 evd_rp->evd_waiters++;
3242 /*
3243 * There are no new events and a timeout was specified.
3244 * Note: for CQ events threshold is 0 but timeout is
3245 * not necessarily 0.
3246 */
3247 while ((evd_rp->evd_newevents == DAPLKA_EVD_NO_EVENTS) &&
3248 timeout) {
3249 retval = DAPLKA_EVD_WAIT(&evd_rp->evd_cv,
3250 &evd_rp->evd_lock, timeout);
3251 if (retval == 0) {
3252 retval = EINTR;
3253 break;
3254 } else if (retval == -1) {
3255 retval = ETIME;
3256 break;
3257 } else {
3258 retval = 0;
3259 continue;
3260 }
3261 }
3262 evd_rp->evd_waiters--;
3263 if (evd_rp->evd_newevents != DAPLKA_EVD_NO_EVENTS) {
3264 /*
3265 * If we got woken up by the CQ handler due to events
3266 * in the CQ. Need to go to userland to check for
3267 * CQ events. Or if we were woken up due to S/W events
3268 */
3269
3270 /* check for userland events only */
3271 if (!(evd_rp->evd_newevents &
3272 ~DAPLKA_EVD_ULAND_EVENTS)) {
3273 evd_rp->evd_newevents = DAPLKA_EVD_NO_EVENTS;
3274 mutex_exit(&evd_rp->evd_lock);
3275 break;
3276 }
3277 /*
3278 * Clear newevents since we are going to loopback
3279 * back and check for both CM and CQ events
3280 */
3281 evd_rp->evd_newevents = DAPLKA_EVD_NO_EVENTS;
3282 } else { /* error */
3283 mutex_exit(&evd_rp->evd_lock);
3284 break;
3285 }
3286 }
3287
3288 maxevent_reached:
3289 args.evp_num_polled = num_events;
3290
3291 /*
3292 * At this point retval might have a value that we want to return
3293 * back to the user. So the copyouts shouldn't tamper retval.
3294 */
3295 if (args.evp_num_polled > 0) { /* copyout the events */
3296 rc = ddi_copyout(evp, args.evp_ep, args.evp_num_polled *
3297 sizeof (dapl_ib_event_t), mode);
3298 if (rc != 0) { /* XXX: we are losing events here */
3299 DERR("event_poll: event array copyout error %d", rc);
3300 retval = EFAULT;
3301 goto cleanup;
3302 }
3303 rc = daplka_event_poll_copyout(&args, arg, mode);
3304 if (rc != 0) { /* XXX: we are losing events here */
3305 DERR("event_poll: copyout error %d\n", rc);
3306 retval = EFAULT;
3307 goto cleanup;
3308 }
3309 }
3310
3311 cleanup:;
3312 if ((max_events > NUM_EVENTS_PER_POLL) && (evp_start != NULL)) {
3313 kmem_free(evp_start, evp_size);
3314 }
3315
3316 if (evd_rp != NULL) {
3317 DAPLKA_RS_UNREF(evd_rp);
3318 }
3319 return (retval);
3320 }
3321
3322 /* ARGSUSED */
3323 static int
3324 daplka_event_wakeup(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3325 cred_t *cred, int *rvalp)
3326 {
3327 dapl_event_wakeup_t args;
3328 daplka_evd_resource_t *evd_rp;
3329 int retval;
3330
3331 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_event_wakeup_t),
3332 mode);
3333 if (retval != 0) {
3334 DERR("event_wakeup: copyin error %d\n", retval);
3335 return (EFAULT);
3336 }
3337
3338 /* get evd resource */
3339 evd_rp = (daplka_evd_resource_t *)
3340 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evw_hkey);
3341 if (evd_rp == NULL) {
3342 DERR("event_wakeup: cannot find evd resource\n");
3343 return (EINVAL);
3344 }
3345 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
3346
3347 daplka_evd_wakeup(evd_rp, NULL, NULL);
3348
3349 DAPLKA_RS_UNREF(evd_rp);
3350
3351 return (retval);
3352 }
3353
3354 /* ARGSUSED */
3355 static int
3356 daplka_evd_modify_cno(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3357 cred_t *cred, int *rvalp)
3358 {
3359 dapl_evd_modify_cno_t args;
3360 daplka_evd_resource_t *evd_rp;
3361 daplka_cno_resource_t *cno_rp;
3362 daplka_cno_resource_t *old_cno_rp;
3363 int retval;
3364
3365 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_modify_cno_t),
3366 mode);
3367 if (retval != 0) {
3368 DERR("evd_modify_cno: copyin error %d\n", retval);
3369 return (EFAULT);
3370 }
3371
3372 /* get evd resource */
3373 evd_rp = (daplka_evd_resource_t *)
3374 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evmc_hkey);
3375 if (evd_rp == NULL) {
3376 DERR("evd_modify_cno: cannot find evd resource\n");
3377 retval = EINVAL;
3378 goto cleanup;
3379 }
3380 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
3381
3382 if (args.evmc_cno_hkey > 0) {
3383 /* get cno resource corresponding to the new CNO */
3384 cno_rp = (daplka_cno_resource_t *)
3385 daplka_hash_lookup(&ia_rp->ia_cno_htbl,
3386 args.evmc_cno_hkey);
3387 if (cno_rp == NULL) {
3388 DERR("evd_modify_cno: cannot find CNO resource\n");
3389 retval = EINVAL;
3390 goto cleanup;
3391 }
3392 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
3393 } else {
3394 cno_rp = NULL;
3395 }
3396
3397 mutex_enter(&evd_rp->evd_lock);
3398 old_cno_rp = evd_rp->evd_cno_res;
3399 evd_rp->evd_cno_res = cno_rp;
3400 mutex_exit(&evd_rp->evd_lock);
3401
3402 /*
3403 * drop the refcnt on the old CNO, the refcnt on the new CNO is
3404 * retained since the evd holds a reference to it.
3405 */
3406 if (old_cno_rp) {
3407 DAPLKA_RS_UNREF(old_cno_rp);
3408 }
3409
3410 cleanup:
3411 if (evd_rp) {
3412 DAPLKA_RS_UNREF(evd_rp);
3413 }
3414
3415 return (retval);
3416 }
3417
3418 /*
3419 * Frees the EVD and associated resources.
3420 * If there are other threads still using this EVD, the destruction
3421 * will defer until the EVD's refcnt drops to zero.
3422 */
3423 /* ARGSUSED */
3424 static int
3425 daplka_evd_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3426 cred_t *cred, int *rvalp)
3427 {
3428 daplka_evd_resource_t *evd_rp = NULL;
3429 daplka_async_evd_hkey_t *curr;
3430 daplka_async_evd_hkey_t *prev;
3431 dapl_evd_free_t args;
3432 int retval = 0;
3433
3434 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_free_t), mode);
3435 if (retval != 0) {
3436 DERR("evd_free: copyin error %d\n", retval);
3437 return (EFAULT);
3438 }
3439 retval = daplka_hash_remove(&ia_rp->ia_evd_htbl, args.evf_hkey,
3440 (void **)&evd_rp);
3441 if (retval != 0 || evd_rp == NULL) {
3442 DERR("evd_free: cannot find evd resource\n");
3443 return (EINVAL);
3444 }
3445 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
3446
3447 /* If this is an async evd remove it from the IA's async evd list */
3448 if (evd_rp->evd_flags & DAT_EVD_ASYNC_FLAG) {
3449 mutex_enter(&ia_rp->ia_lock);
3450 curr = prev = ia_rp->ia_async_evd_hkeys;
3451 while (curr != NULL) {
3452 if (curr->aeh_evd_hkey == args.evf_hkey) {
3453 /* unlink curr from the list */
3454 if (curr == prev) {
3455 /*
3456 * if first element in the list update
3457 * the list head
3458 */
3459 ia_rp->ia_async_evd_hkeys =
3460 curr->aeh_next;
3461 } else {
3462 prev->aeh_next = curr->aeh_next;
3463 }
3464 break;
3465 }
3466 prev = curr;
3467 curr = curr->aeh_next;
3468 }
3469 mutex_exit(&ia_rp->ia_lock);
3470 /* free the curr entry */
3471 kmem_free(curr, sizeof (daplka_async_evd_hkey_t));
3472 }
3473
3474 /* UNREF calls the actual free function when refcnt is zero */
3475 DAPLKA_RS_UNREF(evd_rp);
3476 return (0);
3477 }
3478
3479 /*
3480 * destroys EVD resource.
3481 * called when refcnt drops to zero.
3482 */
3483 static int
3484 daplka_evd_destroy(daplka_resource_t *gen_rp)
3485 {
3486 daplka_evd_resource_t *evd_rp = (daplka_evd_resource_t *)gen_rp;
3487 ibt_status_t status;
3488 daplka_evd_event_t *evt;
3489 ibt_priv_data_len_t len;
3490
3491 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*evd_rp))
3492 D3("evd_destroy: entering, evd_rp 0x%p, rnum %d\n",
3493 evd_rp, DAPLKA_RS_RNUM(evd_rp));
3494 /*
3495 * free CQ
3496 */
3497 if (evd_rp->evd_cq_hdl) {
3498 ibt_set_cq_handler(evd_rp->evd_cq_hdl, NULL, NULL);
3499 mutex_enter(&daplka_dev->daplka_mutex);
3500 ibt_set_cq_private(evd_rp->evd_cq_hdl, NULL);
3501 mutex_exit(&daplka_dev->daplka_mutex);
3502
3503 status = daplka_ibt_free_cq(evd_rp, evd_rp->evd_cq_hdl);
3504 if (status != IBT_SUCCESS) {
3505 DERR("evd_destroy: ibt_free_cq returned %d\n", status);
3506 }
3507 evd_rp->evd_cq_hdl = NULL;
3508 D2("evd_destroy: cq freed, rnum %d\n", DAPLKA_RS_RNUM(evd_rp));
3509 }
3510
3511 /*
3512 * release reference on CNO
3513 */
3514 if (evd_rp->evd_cno_res != NULL) {
3515 mutex_enter(&evd_rp->evd_cno_res->cno_lock);
3516 if (evd_rp->evd_cno_res->cno_evd_cookie ==
3517 evd_rp->evd_cookie) {
3518 evd_rp->evd_cno_res->cno_evd_cookie = 0;
3519 }
3520 mutex_exit(&evd_rp->evd_cno_res->cno_lock);
3521 DAPLKA_RS_UNREF(evd_rp->evd_cno_res);
3522 evd_rp->evd_cno_res = NULL;
3523 }
3524
3525 /*
3526 * discard all remaining events
3527 */
3528 mutex_enter(&evd_rp->evd_lock);
3529 while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_cr_events))) {
3530 D2("evd_destroy: discarding CR event: %d\n",
3531 evt->ee_cmev.ec_cm_ev_type);
3532 len = evt->ee_cmev.ec_cm_ev_priv_data_len;
3533 if (len > 0) {
3534 kmem_free(evt->ee_cmev.ec_cm_ev_priv_data, len);
3535 evt->ee_cmev.ec_cm_ev_priv_data = NULL;
3536 evt->ee_cmev.ec_cm_ev_priv_data_len = 0;
3537 }
3538 kmem_free(evt, sizeof (*evt));
3539 }
3540 ASSERT(evd_rp->evd_cr_events.eel_num_elements == 0);
3541
3542 while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_conn_events))) {
3543 D2("evd_destroy: discarding CONN event: %d\n",
3544 evt->ee_cmev.ec_cm_ev_type);
3545 len = evt->ee_cmev.ec_cm_ev_priv_data_len;
3546 if (len > 0) {
3547 kmem_free(evt->ee_cmev.ec_cm_ev_priv_data, len);
3548 evt->ee_cmev.ec_cm_ev_priv_data = NULL;
3549 evt->ee_cmev.ec_cm_ev_priv_data_len = 0;
3550 }
3551 kmem_free(evt, sizeof (*evt));
3552 }
3553 ASSERT(evd_rp->evd_conn_events.eel_num_elements == 0);
3554
3555 while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_async_events))) {
3556 DERR("evd_destroy: discarding ASYNC event: %d\n",
3557 evt->ee_aev.ibae_type);
3558 kmem_free(evt, sizeof (*evt));
3559 }
3560 ASSERT(evd_rp->evd_async_events.eel_num_elements == 0);
3561 mutex_exit(&evd_rp->evd_lock);
3562
3563 mutex_destroy(&evd_rp->evd_lock);
3564 DAPLKA_RS_FINI(evd_rp);
3565 kmem_free(evd_rp, sizeof (daplka_evd_resource_t));
3566 D3("evd_destroy: exiting, evd_rp 0x%p\n", evd_rp);
3567 return (0);
3568 }
3569
3570 static void
3571 daplka_hash_evd_free(void *obj)
3572 {
3573 daplka_evd_resource_t *evd_rp = (daplka_evd_resource_t *)obj;
3574
3575 ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
3576 DAPLKA_RS_UNREF(evd_rp);
3577 }
3578
3579 /*
3580 * this handler fires when new completions arrive.
3581 */
3582 /* ARGSUSED */
3583 static void
3584 daplka_cq_handler(ibt_cq_hdl_t ibt_cq, void *arg)
3585 {
3586 D3("cq_handler: fired setting evd_newevents\n");
3587 daplka_evd_wakeup((daplka_evd_resource_t *)arg, NULL, NULL);
3588 }
3589
3590 /*
3591 * this routine wakes up a client from evd_wait. if evtq and evt
3592 * are non-null, the event evt will be enqueued prior to waking
3593 * up the client. if the evd is associated with a CNO and if there
3594 * are no waiters on the evd, the CNO will be notified.
3595 */
3596 static void
3597 daplka_evd_wakeup(daplka_evd_resource_t *evd_rp, daplka_evd_event_list_t *evtq,
3598 daplka_evd_event_t *evt)
3599 {
3600 uint32_t waiters = 0;
3601
3602 mutex_enter(&evd_rp->evd_lock);
3603 if (evtq != NULL && evt != NULL) {
3604 ASSERT(evtq == &evd_rp->evd_cr_events ||
3605 evtq == &evd_rp->evd_conn_events ||
3606 evtq == &evd_rp->evd_async_events);
3607 daplka_evd_event_enqueue(evtq, evt);
3608 ASSERT((evtq->eel_event_type == DAPLKA_EVD_CM_EVENTS) ||
3609 (evtq->eel_event_type == DAPLKA_EVD_ASYNC_EVENTS));
3610 evd_rp->evd_newevents |= evtq->eel_event_type;
3611 } else {
3612 evd_rp->evd_newevents |= DAPLKA_EVD_ULAND_EVENTS;
3613 }
3614 waiters = evd_rp->evd_waiters;
3615 cv_broadcast(&evd_rp->evd_cv);
3616 mutex_exit(&evd_rp->evd_lock);
3617
3618 /*
3619 * only wakeup the CNO if there are no waiters on this evd.
3620 */
3621 if (evd_rp->evd_cno_res != NULL && waiters == 0) {
3622 mutex_enter(&evd_rp->evd_cno_res->cno_lock);
3623 evd_rp->evd_cno_res->cno_evd_cookie = evd_rp->evd_cookie;
3624 cv_broadcast(&evd_rp->evd_cno_res->cno_cv);
3625 mutex_exit(&evd_rp->evd_cno_res->cno_lock);
3626 }
3627 }
3628
3629 /*
3630 * daplka_evd_event_enqueue adds elem to the end of the event list
3631 * The caller is expected to acquire appropriate locks before
3632 * calling enqueue
3633 */
3634 static void
3635 daplka_evd_event_enqueue(daplka_evd_event_list_t *evlist,
3636 daplka_evd_event_t *elem)
3637 {
3638 if (evlist->eel_tail) {
3639 evlist->eel_tail->ee_next = elem;
3640 evlist->eel_tail = elem;
3641 } else {
3642 /* list is empty */
3643 ASSERT(evlist->eel_head == NULL);
3644 evlist->eel_head = elem;
3645 evlist->eel_tail = elem;
3646 }
3647 evlist->eel_num_elements++;
3648 }
3649
3650 /*
3651 * daplka_evd_event_dequeue removes and returns the first element of event
3652 * list. NULL is returned if the list is empty. The caller is expected to
3653 * acquire appropriate locks before calling enqueue.
3654 */
3655 static daplka_evd_event_t *
3656 daplka_evd_event_dequeue(daplka_evd_event_list_t *evlist)
3657 {
3658 daplka_evd_event_t *head;
3659
3660 head = evlist->eel_head;
3661 if (head == NULL) {
3662 return (NULL);
3663 }
3664
3665 evlist->eel_head = head->ee_next;
3666 evlist->eel_num_elements--;
3667 /* if it was the last element update the tail pointer too */
3668 if (evlist->eel_head == NULL) {
3669 ASSERT(evlist->eel_num_elements == 0);
3670 evlist->eel_tail = NULL;
3671 }
3672 return (head);
3673 }
3674
3675 /*
3676 * A CNO allows the client to wait for notifications from multiple EVDs.
3677 * To use a CNO, the client needs to follow the procedure below:
3678 * 1. allocate a CNO. this returns a cno_hkey that identifies the CNO.
3679 * 2. create one or more EVDs using the returned cno_hkey.
3680 * 3. call cno_wait. when one of the associated EVDs get notified, the
3681 * CNO will also get notified. cno_wait will then return with a
3682 * evd_cookie identifying the EVD that triggered the event.
3683 *
3684 * A note about cno_wait:
3685 * -unlike a EVD, a CNO does not maintain a queue of notifications. For
3686 * example, suppose multiple EVDs triggered a CNO before the client calls
3687 * cno_wait; when the client calls cno_wait, it will return with the
3688 * evd_cookie that identifies the *last* EVD that triggered the CNO. It
3689 * is the responsibility of the client, upon returning from cno_wait, to
3690 * check on all EVDs that can potentially trigger the CNO. the returned
3691 * evd_cookie is only meant to be a hint. there is no guarantee that the
3692 * EVD identified by the evd_cookie still contains an event or still
3693 * exists by the time cno_wait returns.
3694 */
3695
3696 /*
3697 * allocates a CNO.
3698 * the returned cno_hkey may subsequently be used in evd_create.
3699 */
3700 /* ARGSUSED */
3701 static int
3702 daplka_cno_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3703 cred_t *cred, int *rvalp)
3704 {
3705 dapl_cno_alloc_t args;
3706 daplka_cno_resource_t *cno_rp = NULL;
3707 uint64_t cno_hkey = 0;
3708 boolean_t inserted = B_FALSE;
3709 int retval = 0;
3710
3711 cno_rp = kmem_zalloc(sizeof (*cno_rp), daplka_km_flags);
3712 if (cno_rp == NULL) {
3713 DERR("cno_alloc: cannot allocate cno resource\n");
3714 return (ENOMEM);
3715 }
3716 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cno_rp))
3717 DAPLKA_RS_INIT(cno_rp, DAPL_TYPE_CNO,
3718 DAPLKA_RS_RNUM(ia_rp), daplka_cno_destroy);
3719
3720 mutex_init(&cno_rp->cno_lock, NULL, MUTEX_DRIVER, NULL);
3721 cv_init(&cno_rp->cno_cv, NULL, CV_DRIVER, NULL);
3722 cno_rp->cno_evd_cookie = 0;
3723
3724 /* insert into cno hash table */
3725 retval = daplka_hash_insert(&ia_rp->ia_cno_htbl,
3726 &cno_hkey, (void *)cno_rp);
3727 if (retval != 0) {
3728 DERR("cno_alloc: cannot insert cno resource\n");
3729 goto cleanup;
3730 }
3731 inserted = B_TRUE;
3732 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*cno_rp))
3733
3734 /* return hkey to library */
3735 args.cno_hkey = cno_hkey;
3736
3737 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_cno_alloc_t),
3738 mode);
3739 if (retval != 0) {
3740 DERR("cno_alloc: copyout error %d\n", retval);
3741 retval = EFAULT;
3742 goto cleanup;
3743 }
3744 return (0);
3745
3746 cleanup:;
3747 if (inserted) {
3748 daplka_cno_resource_t *free_rp = NULL;
3749
3750 (void) daplka_hash_remove(&ia_rp->ia_cno_htbl, cno_hkey,
3751 (void **)&free_rp);
3752 if (free_rp != cno_rp) {
3753 DERR("cno_alloc: cannot remove cno\n");
3754 /*
3755 * we can only get here if another thread
3756 * has completed the cleanup in cno_free
3757 */
3758 return (retval);
3759 }
3760 }
3761 DAPLKA_RS_UNREF(cno_rp);
3762 return (retval);
3763 }
3764
3765 /*
3766 * destroys a CNO.
3767 * this gets called when a CNO resource's refcnt drops to zero.
3768 */
3769 static int
3770 daplka_cno_destroy(daplka_resource_t *gen_rp)
3771 {
3772 daplka_cno_resource_t *cno_rp = (daplka_cno_resource_t *)gen_rp;
3773
3774 ASSERT(DAPLKA_RS_REFCNT(cno_rp) == 0);
3775 D2("cno_destroy: entering, cno_rp %p, rnum %d\n",
3776 cno_rp, DAPLKA_RS_RNUM(cno_rp));
3777
3778 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
3779 cv_destroy(&cno_rp->cno_cv);
3780 mutex_destroy(&cno_rp->cno_lock);
3781
3782 DAPLKA_RS_FINI(cno_rp);
3783 kmem_free(cno_rp, sizeof (daplka_cno_resource_t));
3784 D2("cno_destroy: exiting, cno_rp %p\n", cno_rp);
3785 return (0);
3786 }
3787
3788 static void
3789 daplka_hash_cno_free(void *obj)
3790 {
3791 daplka_cno_resource_t *cno_rp = (daplka_cno_resource_t *)obj;
3792
3793 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
3794 DAPLKA_RS_UNREF(cno_rp);
3795 }
3796
3797 /*
3798 * removes the CNO from the cno hash table and frees the CNO
3799 * if there are no references to it. if there are references to
3800 * it, the CNO will be destroyed when the last of the references
3801 * is released. once the CNO is removed from the cno hash table,
3802 * the client will no longer be able to call cno_wait on the CNO.
3803 */
3804 /* ARGSUSED */
3805 static int
3806 daplka_cno_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3807 cred_t *cred, int *rvalp)
3808 {
3809 daplka_cno_resource_t *cno_rp = NULL;
3810 dapl_cno_free_t args;
3811 int retval = 0;
3812
3813 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cno_free_t), mode);
3814 if (retval != 0) {
3815 DERR("cno_free: copyin error %d\n", retval);
3816 return (EINVAL);
3817 }
3818
3819 retval = daplka_hash_remove(&ia_rp->ia_cno_htbl,
3820 args.cnf_hkey, (void **)&cno_rp);
3821 if (retval != 0 || cno_rp == NULL) {
3822 DERR("cno_free: cannot find cno resource\n");
3823 return (EINVAL);
3824 }
3825 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
3826
3827 /* UNREF calls the actual free function when refcnt is zero */
3828 DAPLKA_RS_UNREF(cno_rp);
3829 return (0);
3830 }
3831
3832 /*
3833 * wait for a notification from one of the associated EVDs.
3834 */
3835 /* ARGSUSED */
3836 static int
3837 daplka_cno_wait(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3838 cred_t *cred, int *rvalp)
3839 {
3840 daplka_cno_resource_t *cno_rp = NULL;
3841 dapl_cno_wait_t args;
3842 int retval = 0;
3843 uint64_t evd_cookie = 0;
3844 clock_t timeout, curr_time;
3845
3846 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cno_wait_t), mode);
3847 if (retval != 0) {
3848 DERR("cno_wait: copyin error %d\n", retval);
3849 return (EINVAL);
3850 }
3851 /* get cno resource */
3852 cno_rp = (daplka_cno_resource_t *)
3853 daplka_hash_lookup(&ia_rp->ia_cno_htbl, args.cnw_hkey);
3854 if (cno_rp == NULL) {
3855 DERR("cno_wait: cannot find cno resource\n");
3856 return (EINVAL);
3857 }
3858 ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
3859
3860 curr_time = ddi_get_lbolt();
3861 timeout = curr_time + drv_usectohz(args.cnw_timeout);
3862
3863 /*
3864 * use the max value if we wrapped around
3865 */
3866 if (args.cnw_timeout > 0 && timeout <= curr_time) {
3867 /*
3868 * clock_t (size long) changes between 32 and 64-bit kernels
3869 */
3870 timeout = LONG_MAX >> 4;
3871 }
3872 mutex_enter(&cno_rp->cno_lock);
3873 while (cno_rp->cno_evd_cookie == 0) {
3874 int rval = 0;
3875
3876 rval = cv_timedwait_sig(&cno_rp->cno_cv,
3877 &cno_rp->cno_lock, timeout);
3878 if (rval == 0) {
3879 DERR("cno_wait: interrupted\n");
3880 mutex_exit(&cno_rp->cno_lock);
3881 retval = EINTR;
3882 goto cleanup;
3883 } else if (rval == -1) {
3884 DERR("cno_wait: timed out\n");
3885 mutex_exit(&cno_rp->cno_lock);
3886 retval = ETIME;
3887 goto cleanup;
3888 }
3889 }
3890 evd_cookie = cno_rp->cno_evd_cookie;
3891 cno_rp->cno_evd_cookie = 0;
3892 mutex_exit(&cno_rp->cno_lock);
3893
3894 ASSERT(evd_cookie != 0);
3895 D2("cno_wait: returning evd_cookie 0x%p\n",
3896 (void *)(uintptr_t)evd_cookie);
3897 args.cnw_evd_cookie = evd_cookie;
3898 retval = ddi_copyout((void *)&args, (void *)arg,
3899 sizeof (dapl_cno_wait_t), mode);
3900 if (retval != 0) {
3901 DERR("cno_wait: copyout error %d\n", retval);
3902 retval = EFAULT;
3903 goto cleanup;
3904 }
3905
3906 cleanup:;
3907 if (cno_rp != NULL) {
3908 DAPLKA_RS_UNREF(cno_rp);
3909 }
3910 return (retval);
3911 }
3912
3913 /*
3914 * this function is called by the client when it decides to
3915 * accept a connection request. a connection request is generated
3916 * when the active side generates REQ MAD to a service point on
3917 * the destination node. this causes the CM service handler
3918 * (daplka_cm_service_req) on the passive side to be callee. This
3919 * handler will then enqueue this connection request to the backlog
3920 * array of the service point. A connection event containing the
3921 * backlog array index and connection request private data is passed
3922 * to the client's service point EVD (sp_evd_res). once the event
3923 * is passed up to the userland, the client may examine the request
3924 * to decide whether to call daplka_cr_accept or dapka_cr_reject.
3925 */
3926 /* ARGSUSED */
3927 static int
3928 daplka_cr_accept(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
3929 cred_t *cred, int *rvalp)
3930 {
3931 daplka_ep_resource_t *ep_rp = NULL;
3932 daplka_sp_resource_t *sp_rp = NULL;
3933 dapl_cr_accept_t args;
3934 daplka_sp_conn_pend_t *conn;
3935 ibt_cm_proceed_reply_t proc_reply;
3936 ibt_status_t status;
3937 uint16_t bkl_index;
3938 uint32_t old_state, new_state;
3939 int retval = 0;
3940 void *priv_data = NULL, *sid;
3941
3942 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_accept_t),
3943 mode);
3944 if (retval != 0) {
3945 DERR("cr_accept: copyin error %d\n", retval);
3946 return (EFAULT);
3947 }
3948 if (args.cra_priv_sz > DAPL_MAX_PRIVATE_DATA_SIZE) {
3949 DERR("cr_accept: private data len (%d) exceeded "
3950 "max size %d\n", args.cra_priv_sz,
3951 DAPL_MAX_PRIVATE_DATA_SIZE);
3952 return (EINVAL);
3953 }
3954 priv_data = (args.cra_priv_sz > 0) ? (void *)args.cra_priv : NULL;
3955
3956 D2("cr_accept: priv(0x%p) priv_len(%u) psep(0x%llx)\n", priv_data,
3957 args.cra_priv_sz, (longlong_t)args.cra_bkl_cookie);
3958
3959 /* get sp resource */
3960 sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
3961 args.cra_sp_hkey);
3962 if (sp_rp == NULL) {
3963 DERR("cr_accept: cannot find sp resource\n");
3964 return (EINVAL);
3965 }
3966 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
3967
3968 /* get ep resource */
3969 ep_rp = (daplka_ep_resource_t *)daplka_hash_lookup(&ia_rp->ia_ep_htbl,
3970 args.cra_ep_hkey);
3971 if (ep_rp == NULL) {
3972 DERR("cr_accept: cannot find ep resource\n");
3973 retval = EINVAL;
3974 goto cleanup;
3975 }
3976 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
3977
3978 /*
3979 * accept is only allowed if ep_state is CLOSED.
3980 * note that after this point, the ep_state is frozen
3981 * (i.e. TRANSITIONING) until we transition ep_state
3982 * to ACCEPTING or back to CLOSED if we get an error.
3983 */
3984 new_state = old_state = daplka_ep_get_state(ep_rp);
3985 if (old_state != DAPLKA_EP_STATE_CLOSED) {
3986 DERR("cr_accept: invalid ep state %d\n", old_state);
3987 retval = EINVAL;
3988 goto cleanup;
3989 }
3990
3991 mutex_enter(&sp_rp->sp_lock);
3992 bkl_index = DAPLKA_GET_PSEP_INDEX(args.cra_bkl_cookie);
3993 /*
3994 * make sure the backlog index is not bogus.
3995 */
3996 if (bkl_index >= sp_rp->sp_backlog_size) {
3997 DERR("cr_accept: invalid backlog index 0x%llx %d\n",
3998 (longlong_t)args.cra_bkl_cookie, bkl_index);
3999 mutex_exit(&sp_rp->sp_lock);
4000 retval = EINVAL;
4001 goto cleanup;
4002 }
4003 /*
4004 * make sure the backlog index indeed refers
4005 * to a pending connection.
4006 */
4007 conn = &sp_rp->sp_backlog[bkl_index];
4008 if (conn->spcp_state != DAPLKA_SPCP_PENDING) {
4009 DERR("cr_accept: invalid conn state %d\n",
4010 conn->spcp_state);
4011 mutex_exit(&sp_rp->sp_lock);
4012 retval = EINVAL;
4013 goto cleanup;
4014 }
4015 if (conn->spcp_sid == NULL) {
4016 DERR("cr_accept: sid == NULL\n");
4017 mutex_exit(&sp_rp->sp_lock);
4018 retval = EINVAL;
4019 goto cleanup;
4020 }
4021 if (ep_rp->ep_chan_hdl == NULL) {
4022 /*
4023 * a ep_rp with a NULL chan_hdl is impossible.
4024 */
4025 DERR("cr_accept: ep_chan_hdl == NULL\n");
4026 mutex_exit(&sp_rp->sp_lock);
4027 ASSERT(B_FALSE);
4028 retval = EINVAL;
4029 goto cleanup;
4030 }
4031 proc_reply.rep.cm_channel = ep_rp->ep_chan_hdl;
4032 proc_reply.rep.cm_rdma_ra_out = conn->spcp_rdma_ra_out;
4033 proc_reply.rep.cm_rdma_ra_in = conn->spcp_rdma_ra_in;
4034 proc_reply.rep.cm_rnr_retry_cnt = IBT_RNR_INFINITE_RETRY;
4035 sid = conn->spcp_sid;
4036
4037 /*
4038 * this clears our slot in the backlog array.
4039 * this slot may now be used by other pending connections.
4040 */
4041 conn->spcp_sid = NULL;
4042 conn->spcp_state = DAPLKA_SPCP_INIT;
4043 conn->spcp_req_len = 0;
4044 mutex_exit(&sp_rp->sp_lock);
4045
4046 /*
4047 * Set the unique cookie corresponding to the CR to this EP
4048 * so that is can be used in passive side CM callbacks
4049 */
4050 ep_rp->ep_psep_cookie = args.cra_bkl_cookie;
4051
4052 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid, IBT_CM_ACCEPT,
4053 &proc_reply, priv_data, (ibt_priv_data_len_t)args.cra_priv_sz);
4054
4055 if (status != IBT_SUCCESS) {
4056 DERR("cr_accept: ibt_cm_proceed returned %d\n", status);
4057 *rvalp = (int)status;
4058 retval = 0;
4059 }
4060 /*
4061 * note that the CM handler may actually be called at this
4062 * point. but since ep_state is still in TRANSITIONING, the
4063 * handler will wait until we transition to ACCEPTING. this
4064 * prevents the case where we set ep_state to ACCEPTING after
4065 * daplka_service_conn_est sets ep_state to CONNECTED.
4066 */
4067 new_state = DAPLKA_EP_STATE_ACCEPTING;
4068
4069 cleanup:;
4070 if (sp_rp != NULL) {
4071 DAPLKA_RS_UNREF(sp_rp);
4072 }
4073 if (ep_rp != NULL) {
4074 daplka_ep_set_state(ep_rp, old_state, new_state);
4075 DAPLKA_RS_UNREF(ep_rp);
4076 }
4077 return (retval);
4078 }
4079
4080 /*
4081 * this function is called by the client to reject a
4082 * connection request.
4083 */
4084 /* ARGSUSED */
4085 static int
4086 daplka_cr_reject(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
4087 cred_t *cred, int *rvalp)
4088 {
4089 dapl_cr_reject_t args;
4090 daplka_sp_resource_t *sp_rp = NULL;
4091 daplka_sp_conn_pend_t *conn;
4092 ibt_cm_proceed_reply_t proc_reply;
4093 ibt_cm_status_t proc_status;
4094 ibt_status_t status;
4095 uint16_t bkl_index;
4096 int retval = 0;
4097 void *sid;
4098
4099 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_reject_t),
4100 mode);
4101 if (retval != 0) {
4102 DERR("cr_reject: copyin error %d\n", retval);
4103 return (EFAULT);
4104 }
4105 /* get sp resource */
4106 sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
4107 args.crr_sp_hkey);
4108 if (sp_rp == NULL) {
4109 DERR("cr_reject: cannot find sp resource\n");
4110 return (EINVAL);
4111 }
4112 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
4113
4114 D2("cr_reject: psep(0x%llx)\n", (longlong_t)args.crr_bkl_cookie);
4115
4116 mutex_enter(&sp_rp->sp_lock);
4117 bkl_index = DAPLKA_GET_PSEP_INDEX(args.crr_bkl_cookie);
4118 /*
4119 * make sure the backlog index is not bogus.
4120 */
4121 if (bkl_index >= sp_rp->sp_backlog_size) {
4122 DERR("cr_reject: invalid backlog index 0x%llx %d\n",
4123 (longlong_t)args.crr_bkl_cookie, bkl_index);
4124 mutex_exit(&sp_rp->sp_lock);
4125 retval = EINVAL;
4126 goto cleanup;
4127 }
4128 /*
4129 * make sure the backlog index indeed refers
4130 * to a pending connection.
4131 */
4132 conn = &sp_rp->sp_backlog[bkl_index];
4133 if (conn->spcp_state != DAPLKA_SPCP_PENDING) {
4134 DERR("cr_reject: invalid conn state %d\n",
4135 conn->spcp_state);
4136 mutex_exit(&sp_rp->sp_lock);
4137 retval = EINVAL;
4138 goto cleanup;
4139 }
4140 if (conn->spcp_sid == NULL) {
4141 DERR("cr_reject: sid == NULL\n");
4142 mutex_exit(&sp_rp->sp_lock);
4143 retval = EINVAL;
4144 goto cleanup;
4145 }
4146 bzero(&proc_reply, sizeof (proc_reply));
4147 sid = conn->spcp_sid;
4148
4149 /*
4150 * this clears our slot in the backlog array.
4151 * this slot may now be used by other pending connections.
4152 */
4153 conn->spcp_sid = NULL;
4154 conn->spcp_state = DAPLKA_SPCP_INIT;
4155 conn->spcp_req_len = 0;
4156
4157 switch (args.crr_reason) {
4158 case DAPL_IB_CM_REJ_REASON_CONSUMER_REJ:
4159 /* results in IBT_CM_CONSUMER as the reason for reject */
4160 proc_status = IBT_CM_REJECT;
4161 break;
4162 case DAPL_IB_CME_LOCAL_FAILURE:
4163 /*FALLTHRU*/
4164 case DAPL_IB_CME_DESTINATION_UNREACHABLE:
4165 /* results in IBT_CM_NO_RESC as the reason for reject */
4166 proc_status = IBT_CM_NO_RESOURCE;
4167 break;
4168 default:
4169 /* unexpect reason code */
4170 ASSERT(!"unexpected reject reason code");
4171 proc_status = IBT_CM_NO_RESOURCE;
4172 break;
4173 }
4174
4175 mutex_exit(&sp_rp->sp_lock);
4176
4177 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid, proc_status,
4178 &proc_reply, NULL, 0);
4179
4180 if (status != IBT_SUCCESS) {
4181 DERR("cr_reject: ibt_cm_proceed returned %d\n", status);
4182 *rvalp = (int)status;
4183 retval = 0;
4184 }
4185
4186 cleanup:;
4187 if (sp_rp != NULL) {
4188 DAPLKA_RS_UNREF(sp_rp);
4189 }
4190 return (retval);
4191 }
4192
4193
4194 /*
4195 * daplka_sp_match is used by daplka_hash_walk for finding SPs
4196 */
4197 typedef struct daplka_sp_match_s {
4198 uint64_t spm_conn_qual;
4199 daplka_sp_resource_t *spm_sp_rp;
4200 } daplka_sp_match_t;
4201 _NOTE(SCHEME_PROTECTS_DATA("daplka", daplka_sp_match_s::spm_sp_rp))
4202
4203 static int
4204 daplka_sp_match(void *objp, void *arg)
4205 {
4206 daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)objp;
4207
4208 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
4209 if (sp_rp->sp_conn_qual ==
4210 ((daplka_sp_match_t *)arg)->spm_conn_qual) {
4211 ((daplka_sp_match_t *)arg)->spm_sp_rp = sp_rp;
4212 D2("daplka_sp_match: found sp, conn_qual %016llu\n",
4213 (longlong_t)((daplka_sp_match_t *)arg)->spm_conn_qual);
4214 DAPLKA_RS_REF(sp_rp);
4215 return (1);
4216 }
4217 return (0);
4218 }
4219
4220 /*
4221 * cr_handoff allows the client to handoff a connection request from
4222 * one service point to another.
4223 */
4224 /* ARGSUSED */
4225 static int
4226 daplka_cr_handoff(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
4227 cred_t *cred, int *rvalp)
4228 {
4229 dapl_cr_handoff_t args;
4230 daplka_sp_resource_t *sp_rp = NULL, *new_sp_rp = NULL;
4231 daplka_sp_conn_pend_t *conn;
4232 daplka_sp_match_t sp_match;
4233 ibt_cm_event_t fake_event;
4234 ibt_cm_status_t cm_status;
4235 ibt_status_t status;
4236 uint16_t bkl_index;
4237 void *sid, *priv = NULL;
4238 int retval = 0, priv_len = 0;
4239
4240 D3("cr_handoff: entering\n");
4241 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_handoff_t),
4242 mode);
4243 if (retval != 0) {
4244 DERR("cr_handoff: copyin error %d\n", retval);
4245 return (EFAULT);
4246 }
4247 /* get sp resource */
4248 sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
4249 args.crh_sp_hkey);
4250 if (sp_rp == NULL) {
4251 DERR("cr_handoff: cannot find sp resource\n");
4252 return (EINVAL);
4253 }
4254 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
4255
4256 /*
4257 * find the destination service point.
4258 */
4259 sp_match.spm_conn_qual = args.crh_conn_qual;
4260 sp_match.spm_sp_rp = NULL;
4261 daplka_hash_walk(&daplka_global_sp_htbl, daplka_sp_match,
4262 (void *)&sp_match, RW_READER);
4263
4264 /*
4265 * return if we cannot find the service point
4266 */
4267 if (sp_match.spm_sp_rp == NULL) {
4268 DERR("cr_handoff: new sp not found, conn qual = %llu\n",
4269 (longlong_t)args.crh_conn_qual);
4270 retval = EINVAL;
4271 goto cleanup;
4272 }
4273 new_sp_rp = sp_match.spm_sp_rp;
4274
4275 /*
4276 * the spec does not discuss the security implications of this
4277 * function. to be safe, we currently only allow processes
4278 * owned by the same user to handoff connection requests
4279 * to each other.
4280 */
4281 if (crgetruid(cred) != new_sp_rp->sp_ruid) {
4282 DERR("cr_handoff: permission denied\n");
4283 retval = EPERM;
4284 goto cleanup;
4285 }
4286
4287 D2("cr_handoff: psep(0x%llx)\n", (longlong_t)args.crh_bkl_cookie);
4288
4289 mutex_enter(&sp_rp->sp_lock);
4290 bkl_index = DAPLKA_GET_PSEP_INDEX(args.crh_bkl_cookie);
4291 /*
4292 * make sure the backlog index is not bogus.
4293 */
4294 if (bkl_index >= sp_rp->sp_backlog_size) {
4295 DERR("cr_handoff: invalid backlog index 0x%llx %d\n",
4296 (longlong_t)args.crh_bkl_cookie, bkl_index);
4297 mutex_exit(&sp_rp->sp_lock);
4298 retval = EINVAL;
4299 goto cleanup;
4300 }
4301 /*
4302 * make sure the backlog index indeed refers
4303 * to a pending connection.
4304 */
4305 conn = &sp_rp->sp_backlog[bkl_index];
4306 if (conn->spcp_state != DAPLKA_SPCP_PENDING) {
4307 DERR("cr_handoff: invalid conn state %d\n",
4308 conn->spcp_state);
4309 mutex_exit(&sp_rp->sp_lock);
4310 retval = EINVAL;
4311 goto cleanup;
4312 }
4313 if (conn->spcp_sid == NULL) {
4314 DERR("cr_handoff: sid == NULL\n");
4315 mutex_exit(&sp_rp->sp_lock);
4316 retval = EINVAL;
4317 goto cleanup;
4318 }
4319 sid = conn->spcp_sid;
4320 priv = NULL;
4321 priv_len = conn->spcp_req_len;
4322 if (priv_len > 0) {
4323 priv = kmem_zalloc(priv_len, daplka_km_flags);
4324 if (priv == NULL) {
4325 mutex_exit(&sp_rp->sp_lock);
4326 retval = ENOMEM;
4327 goto cleanup;
4328 }
4329 bcopy(conn->spcp_req_data, priv, priv_len);
4330 }
4331 /*
4332 * this clears our slot in the backlog array.
4333 * this slot may now be used by other pending connections.
4334 */
4335 conn->spcp_sid = NULL;
4336 conn->spcp_state = DAPLKA_SPCP_INIT;
4337 conn->spcp_req_len = 0;
4338 mutex_exit(&sp_rp->sp_lock);
4339
4340 /* fill fake_event and call service_req handler */
4341 bzero(&fake_event, sizeof (fake_event));
4342 fake_event.cm_type = IBT_CM_EVENT_REQ_RCV;
4343 fake_event.cm_session_id = sid;
4344 fake_event.cm_priv_data_len = priv_len;
4345 fake_event.cm_priv_data = priv;
4346
4347 cm_status = daplka_cm_service_req(new_sp_rp,
4348 &fake_event, NULL, priv, (ibt_priv_data_len_t)priv_len);
4349 if (cm_status != IBT_CM_DEFER) {
4350 ibt_cm_proceed_reply_t proc_reply;
4351
4352 DERR("cr_handoff: service_req returned %d\n", cm_status);
4353 /*
4354 * if for some reason cm_service_req failed, we
4355 * reject the connection.
4356 */
4357 bzero(&proc_reply, sizeof (proc_reply));
4358
4359 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid,
4360 IBT_CM_NO_RESOURCE, &proc_reply, NULL, 0);
4361 if (status != IBT_SUCCESS) {
4362 DERR("cr_handoff: ibt_cm_proceed returned %d\n",
4363 status);
4364 }
4365 *rvalp = (int)status;
4366 retval = 0;
4367 }
4368
4369 cleanup:;
4370 if (priv_len > 0 && priv != NULL) {
4371 kmem_free(priv, priv_len);
4372 }
4373 if (new_sp_rp != NULL) {
4374 DAPLKA_RS_UNREF(new_sp_rp);
4375 }
4376 if (sp_rp != NULL) {
4377 DAPLKA_RS_UNREF(sp_rp);
4378 }
4379 D3("cr_handoff: exiting\n");
4380 return (retval);
4381 }
4382
4383 /*
4384 * returns a list of hca attributes
4385 */
4386 /* ARGSUSED */
4387 static int
4388 daplka_ia_query(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
4389 cred_t *cred, int *rvalp)
4390 {
4391 dapl_ia_query_t args;
4392 int retval;
4393 ibt_hca_attr_t *hcap;
4394
4395 hcap = &ia_rp->ia_hca->hca_attr;
4396
4397 /*
4398 * Take the ibt_hca_attr_t and stuff them into dapl_hca_attr_t
4399 */
4400 args.hca_attr.dhca_vendor_id = hcap->hca_vendor_id;
4401 args.hca_attr.dhca_device_id = hcap->hca_device_id;
4402 args.hca_attr.dhca_version_id = hcap->hca_version_id;
4403 args.hca_attr.dhca_max_chans = hcap->hca_max_chans;
4404 args.hca_attr.dhca_max_chan_sz = hcap->hca_max_chan_sz;
4405 args.hca_attr.dhca_max_sgl = hcap->hca_max_sgl;
4406 args.hca_attr.dhca_max_cq = hcap->hca_max_cq;
4407 args.hca_attr.dhca_max_cq_sz = hcap->hca_max_cq_sz;
4408 args.hca_attr.dhca_max_memr = hcap->hca_max_memr;
4409 args.hca_attr.dhca_max_memr_len = hcap->hca_max_memr_len;
4410 args.hca_attr.dhca_max_mem_win = hcap->hca_max_mem_win;
4411 args.hca_attr.dhca_max_rdma_in_chan = hcap->hca_max_rdma_in_chan;
4412 args.hca_attr.dhca_max_rdma_out_chan = hcap->hca_max_rdma_out_chan;
4413 args.hca_attr.dhca_max_partitions = hcap->hca_max_partitions;
4414 args.hca_attr.dhca_nports = hcap->hca_nports;
4415 args.hca_attr.dhca_node_guid = hcap->hca_node_guid;
4416 args.hca_attr.dhca_max_pd = hcap->hca_max_pd;
4417 args.hca_attr.dhca_max_srqs = hcap->hca_max_srqs;
4418 args.hca_attr.dhca_max_srqs_sz = hcap->hca_max_srqs_sz;
4419 args.hca_attr.dhca_max_srq_sgl = hcap->hca_max_srq_sgl;
4420
4421 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_ia_query_t),
4422 mode);
4423 if (retval != 0) {
4424 DERR("ia_query: copyout error %d\n", retval);
4425 return (EFAULT);
4426 }
4427 return (0);
4428 }
4429
4430 /*
4431 * This routine is passed to hash walk in the daplka_pre_mr_cleanup_callback,
4432 * it frees the mw embedded in the mw resource object.
4433 */
4434
4435 /* ARGSUSED */
4436 static int
4437 daplka_mr_cb_freemw(void *objp, void *arg)
4438 {
4439 daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)objp;
4440 ibt_mw_hdl_t mw_hdl;
4441 ibt_status_t status;
4442
4443 D3("mr_cb_freemw: entering, mw_rp 0x%p\n", mw_rp);
4444 DAPLKA_RS_REF(mw_rp);
4445
4446 mutex_enter(&mw_rp->mw_lock);
4447 mw_hdl = mw_rp->mw_hdl;
4448 /*
4449 * we set mw_hdl to NULL so it won't get freed again
4450 */
4451 mw_rp->mw_hdl = NULL;
4452 mutex_exit(&mw_rp->mw_lock);
4453
4454 if (mw_hdl != NULL) {
4455 status = daplka_ibt_free_mw(mw_rp, mw_rp->mw_hca_hdl, mw_hdl);
4456 if (status != IBT_SUCCESS) {
4457 DERR("mr_cb_freemw: ibt_free_mw returned %d\n", status);
4458 }
4459 D3("mr_cb_freemw: mw freed\n");
4460 }
4461
4462 DAPLKA_RS_UNREF(mw_rp);
4463 return (0);
4464 }
4465
4466 /*
4467 * This routine is called from HCA driver's umem lock undo callback
4468 * when the memory associated with an MR is being unmapped. In this callback
4469 * we free all the MW associated with the IA and post an unaffiliated
4470 * async event to tell the app that there was a catastrophic event.
4471 * This allows the HCA to deregister the MR in its callback processing.
4472 */
4473 static void
4474 daplka_pre_mr_cleanup_callback(void *arg1, void *arg2 /*ARGSUSED*/)
4475 {
4476 daplka_mr_resource_t *mr_rp;
4477 daplka_ia_resource_t *ia_rp;
4478 #ifdef _THROW_ASYNC_EVENT_FROM_MRUNLOCKCB
4479 ibt_async_event_t event;
4480 ibt_hca_attr_t *hca_attrp;
4481 #endif
4482 minor_t rnum;
4483
4484 mr_rp = (daplka_mr_resource_t *)arg1;
4485 rnum = DAPLKA_RS_RNUM(mr_rp);
4486 daplka_shared_mr_free(mr_rp);
4487
4488 ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(rnum);
4489 if (ia_rp == NULL) {
4490 DERR("daplka_mr_unlock_callback: resource not found, rnum %d\n",
4491 rnum);
4492 return;
4493 }
4494
4495 DERR("daplka_mr_unlock_callback: resource(%p) rnum(%d)\n", ia_rp, rnum);
4496
4497 mutex_enter(&ia_rp->ia_lock);
4498 /*
4499 * MW is being alloced OR MW freeze has already begun. In
4500 * both these cases we wait for that to complete before
4501 * continuing.
4502 */
4503 while ((ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS) ||
4504 (ia_rp->ia_state == DAPLKA_IA_MW_FREEZE_IN_PROGRESS)) {
4505 cv_wait(&ia_rp->ia_cv, &ia_rp->ia_lock);
4506 }
4507
4508 switch (ia_rp->ia_state) {
4509 case DAPLKA_IA_INIT:
4510 ia_rp->ia_state = DAPLKA_IA_MW_FREEZE_IN_PROGRESS;
4511 mutex_exit(&ia_rp->ia_lock);
4512 break;
4513 case DAPLKA_IA_MW_FROZEN:
4514 /* the mw on this ia have been freed */
4515 D2("daplka_mr_unlock_callback: ia_state %d nothing to do\n",
4516 ia_rp->ia_state);
4517 mutex_exit(&ia_rp->ia_lock);
4518 goto cleanup;
4519 default:
4520 ASSERT(!"daplka_mr_unlock_callback: IA state invalid");
4521 DERR("daplka_mr_unlock_callback: invalid ia_state %d\n",
4522 ia_rp->ia_state);
4523 mutex_exit(&ia_rp->ia_lock);
4524 goto cleanup;
4525 }
4526
4527 /*
4528 * Walk the mw hash table and free the mws. Acquire a writer
4529 * lock since we don't want anyone else traversing this tree
4530 * while we are freeing the MW.
4531 */
4532 daplka_hash_walk(&ia_rp->ia_mw_htbl, daplka_mr_cb_freemw, NULL,
4533 RW_WRITER);
4534
4535 mutex_enter(&ia_rp->ia_lock);
4536 ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_FREEZE_IN_PROGRESS);
4537 ia_rp->ia_state = DAPLKA_IA_MW_FROZEN;
4538 cv_broadcast(&ia_rp->ia_cv);
4539 mutex_exit(&ia_rp->ia_lock);
4540
4541 /*
4542 * Currently commented out because Oracle skgxp is incapable
4543 * of handling async events correctly.
4544 */
4545 #ifdef _THROW_ASYNC_EVENT_FROM_MRUNLOCKCB
4546 /*
4547 * Enqueue an unaffiliated async error event to indicate this
4548 * IA has encountered a problem that caused the MW to freed up
4549 */
4550
4551 /* Create a fake event, only relevant field is the hca_guid */
4552 bzero(&event, sizeof (ibt_async_event_t));
4553 hca_attrp = &ia_rp->ia_hca->hca_attr;
4554 event.ev_hca_guid = hca_attrp->hca_node_guid;
4555
4556 daplka_async_event_create(IBT_ERROR_LOCAL_CATASTROPHIC, &event, 0,
4557 ia_rp);
4558 #endif /* _THROW_ASYNC_EVENT_FROM_MRUNLOCKCB */
4559
4560 cleanup:;
4561 D2("daplka_mr_unlock_callback: resource(%p) done\n", ia_rp);
4562 DAPLKA_RS_UNREF(ia_rp);
4563 }
4564
4565 /*
4566 * registers a memory region.
4567 * memory locking will be done by the HCA driver.
4568 */
4569 /* ARGSUSED */
4570 static int
4571 daplka_mr_register(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
4572 cred_t *cred, int *rvalp)
4573 {
4574 boolean_t inserted = B_FALSE;
4575 daplka_mr_resource_t *mr_rp;
4576 daplka_pd_resource_t *pd_rp;
4577 dapl_mr_register_t args;
4578 ibt_mr_data_in_t mr_cb_data_in;
4579 uint64_t mr_hkey = 0;
4580 ibt_status_t status;
4581 int retval;
4582
4583 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_register_t),
4584 mode);
4585 if (retval != 0) {
4586 DERR("mr_register: copyin error %d\n", retval);
4587 return (EINVAL);
4588 }
4589 mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
4590 if (mr_rp == NULL) {
4591 DERR("mr_register: cannot allocate mr resource\n");
4592 return (ENOMEM);
4593 }
4594 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
4595 DAPLKA_RS_INIT(mr_rp, DAPL_TYPE_MR,
4596 DAPLKA_RS_RNUM(ia_rp), daplka_mr_destroy);
4597
4598 mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
4599 mr_rp->mr_hca = ia_rp->ia_hca;
4600 mr_rp->mr_hca_hdl = ia_rp->ia_hca_hdl;
4601 mr_rp->mr_next = NULL;
4602 mr_rp->mr_shared_mr = NULL;
4603
4604 /* get pd handle */
4605 pd_rp = (daplka_pd_resource_t *)
4606 daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mr_pd_hkey);
4607 if (pd_rp == NULL) {
4608 DERR("mr_register: cannot find pd resource\n");
4609 retval = EINVAL;
4610 goto cleanup;
4611 }
4612 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
4613 mr_rp->mr_pd_res = pd_rp;
4614
4615 mr_rp->mr_attr.mr_vaddr = args.mr_vaddr;
4616 mr_rp->mr_attr.mr_len = args.mr_len;
4617 mr_rp->mr_attr.mr_as = curproc->p_as;
4618 mr_rp->mr_attr.mr_flags = args.mr_flags | IBT_MR_NOSLEEP;
4619
4620 D3("mr_register: mr_vaddr %p, mr_len %llu, mr_flags 0x%x\n",
4621 (void *)(uintptr_t)mr_rp->mr_attr.mr_vaddr,
4622 (longlong_t)mr_rp->mr_attr.mr_len,
4623 mr_rp->mr_attr.mr_flags);
4624
4625 status = daplka_ibt_register_mr(mr_rp, ia_rp->ia_hca_hdl,
4626 mr_rp->mr_pd_res->pd_hdl, &mr_rp->mr_attr, &mr_rp->mr_hdl,
4627 &mr_rp->mr_desc);
4628
4629 if (status != IBT_SUCCESS) {
4630 DERR("mr_register: ibt_register_mr error %d\n", status);
4631 *rvalp = (int)status;
4632 retval = 0;
4633 goto cleanup;
4634 }
4635
4636 mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
4637 mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
4638 mr_cb_data_in.mr_arg1 = (void *)mr_rp;
4639 mr_cb_data_in.mr_arg2 = NULL;
4640
4641 /* Pass the service driver mr cleanup handler to the hca driver */
4642 status = ibt_ci_data_in(ia_rp->ia_hca_hdl,
4643 IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
4644 &mr_cb_data_in, sizeof (mr_cb_data_in));
4645
4646 if (status != IBT_SUCCESS) {
4647 DERR("mr_register: ibt_ci_data_in error(%d) ver(%d)",
4648 status, mr_cb_data_in.mr_rev);
4649 *rvalp = (int)status;
4650 retval = 0;
4651 goto cleanup;
4652 }
4653
4654 /* insert into mr hash table */
4655 retval = daplka_hash_insert(&ia_rp->ia_mr_htbl,
4656 &mr_hkey, (void *)mr_rp);
4657 if (retval != 0) {
4658 DERR("mr_register: cannot insert mr resource into mr_htbl\n");
4659 goto cleanup;
4660 }
4661 inserted = B_TRUE;
4662 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mr_rp))
4663
4664 args.mr_lkey = mr_rp->mr_desc.md_lkey;
4665 args.mr_rkey = mr_rp->mr_desc.md_rkey;
4666 args.mr_hkey = mr_hkey;
4667
4668 retval = ddi_copyout((void *)&args, (void *)arg,
4669 sizeof (dapl_mr_register_t), mode);
4670 if (retval != 0) {
4671 DERR("mr_register: copyout error %d\n", retval);
4672 retval = EFAULT;
4673 goto cleanup;
4674 }
4675 return (0);
4676
4677 cleanup:;
4678 if (inserted) {
4679 daplka_mr_resource_t *free_rp = NULL;
4680
4681 (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
4682 (void **)&free_rp);
4683 if (free_rp != mr_rp) {
4684 DERR("mr_register: cannot remove mr from hash table\n");
4685 /*
4686 * we can only get here if another thread
4687 * has completed the cleanup in mr_deregister
4688 */
4689 return (retval);
4690 }
4691 }
4692 DAPLKA_RS_UNREF(mr_rp);
4693 return (retval);
4694 }
4695
4696 /*
4697 * registers a shared memory region.
4698 * the client calls this function with the intention to share the memory
4699 * region with other clients. it is assumed that, prior to calling this
4700 * function, the client(s) are already sharing parts of their address
4701 * space using a mechanism such as SYSV shared memory. the first client
4702 * that calls this function will create and insert a daplka_shared_mr_t
4703 * object into the global daplka_shared_mr_tree. this shared mr object
4704 * will be identified by a unique 40-byte key and will maintain a list
4705 * of mr resources. every time this function gets called with the same
4706 * 40-byte key, a new mr resource (containing a new mr handle generated
4707 * by ibt_register_mr or ibt_register_shared_mr) is created and inserted
4708 * into this list. similarly, every time a shared mr gets deregistered
4709 * or invalidated by a callback, the mr resource gets removed from this
4710 * list. the shared mr object has a reference count. when it drops to
4711 * zero, the shared mr object will be removed from the global avl tree
4712 * and be freed.
4713 */
4714 /* ARGSUSED */
4715 static int
4716 daplka_mr_register_shared(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
4717 cred_t *cred, int *rvalp)
4718 {
4719 dapl_mr_register_shared_t args;
4720 daplka_shared_mr_t *smrp = NULL;
4721 daplka_shared_mr_t tmp_smr;
4722 ibt_mr_data_in_t mr_cb_data_in;
4723 avl_index_t where;
4724 boolean_t inserted = B_FALSE;
4725 daplka_mr_resource_t *mr_rp = NULL;
4726 daplka_pd_resource_t *pd_rp;
4727 uint64_t mr_hkey = 0;
4728 ibt_status_t status;
4729 int retval;
4730
4731 retval = ddi_copyin((void *)arg, &args,
4732 sizeof (dapl_mr_register_shared_t), mode);
4733 if (retval != 0) {
4734 DERR("mr_register_shared: copyin error %d\n", retval);
4735 return (EINVAL);
4736 }
4737
4738 mutex_enter(&daplka_shared_mr_lock);
4739 /*
4740 * find smrp from the global avl tree.
4741 * the 40-byte key is used as the lookup key.
4742 */
4743 tmp_smr.smr_cookie = args.mrs_shm_cookie;
4744 smrp = (daplka_shared_mr_t *)
4745 avl_find(&daplka_shared_mr_tree, &tmp_smr, &where);
4746 if (smrp != NULL) {
4747 D2("mr_register_shared: smrp 0x%p, found cookie:\n"
4748 "0x%016llx%016llx%016llx%016llx%016llx\n", smrp,
4749 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[4],
4750 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[3],
4751 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[2],
4752 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[1],
4753 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[0]);
4754
4755 /*
4756 * if the smrp exists, other threads could still be
4757 * accessing it. we wait until they are done before
4758 * we continue.
4759 */
4760 smrp->smr_refcnt++;
4761 while (smrp->smr_state == DAPLKA_SMR_TRANSITIONING) {
4762 D2("mr_register_shared: smrp 0x%p, "
4763 "waiting in transitioning state, refcnt %d\n",
4764 smrp, smrp->smr_refcnt);
4765 cv_wait(&smrp->smr_cv, &daplka_shared_mr_lock);
4766 }
4767 ASSERT(smrp->smr_state == DAPLKA_SMR_READY);
4768 D2("mr_register_shared: smrp 0x%p, refcnt %d, ready\n",
4769 smrp, smrp->smr_refcnt);
4770
4771 /*
4772 * we set smr_state to TRANSITIONING to temporarily
4773 * prevent other threads from trying to access smrp.
4774 */
4775 smrp->smr_state = DAPLKA_SMR_TRANSITIONING;
4776 } else {
4777 D2("mr_register_shared: cannot find cookie:\n"
4778 "0x%016llx%016llx%016llx%016llx%016llx\n",
4779 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[4],
4780 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[3],
4781 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[2],
4782 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[1],
4783 (longlong_t)tmp_smr.smr_cookie.mc_uint_arr[0]);
4784
4785 /*
4786 * if we cannot find smrp, we need to create and
4787 * insert one into daplka_shared_mr_tree
4788 */
4789 smrp = kmem_zalloc(sizeof (daplka_shared_mr_t),
4790 daplka_km_flags);
4791 if (smrp == NULL) {
4792 retval = ENOMEM;
4793 mutex_exit(&daplka_shared_mr_lock);
4794 goto cleanup;
4795 }
4796 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*smrp))
4797 smrp->smr_refcnt = 1;
4798 smrp->smr_cookie = args.mrs_shm_cookie;
4799 smrp->smr_state = DAPLKA_SMR_TRANSITIONING;
4800 smrp->smr_mr_list = NULL;
4801 cv_init(&smrp->smr_cv, NULL, CV_DRIVER, NULL);
4802 avl_insert(&daplka_shared_mr_tree, smrp, where);
4803 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*smrp))
4804 }
4805 mutex_exit(&daplka_shared_mr_lock);
4806
4807 mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
4808 if (mr_rp == NULL) {
4809 DERR("mr_register_shared: cannot allocate mr resource\n");
4810 goto cleanup;
4811 }
4812 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
4813 DAPLKA_RS_INIT(mr_rp, DAPL_TYPE_MR,
4814 DAPLKA_RS_RNUM(ia_rp), daplka_mr_destroy);
4815
4816 mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
4817 mr_rp->mr_hca = ia_rp->ia_hca;
4818 mr_rp->mr_hca_hdl = ia_rp->ia_hca_hdl;
4819 mr_rp->mr_next = NULL;
4820 mr_rp->mr_shared_mr = NULL;
4821
4822 /* get pd handle */
4823 pd_rp = (daplka_pd_resource_t *)
4824 daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mrs_pd_hkey);
4825 if (pd_rp == NULL) {
4826 DERR("mr_register_shared: cannot find pd resource\n");
4827 retval = EINVAL;
4828 goto cleanup;
4829 }
4830 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
4831 mr_rp->mr_pd_res = pd_rp;
4832
4833 mr_rp->mr_attr.mr_vaddr = args.mrs_vaddr;
4834 mr_rp->mr_attr.mr_len = args.mrs_len;
4835 mr_rp->mr_attr.mr_flags = args.mrs_flags | IBT_MR_NOSLEEP;
4836 mr_rp->mr_attr.mr_as = curproc->p_as;
4837
4838 D2("mr_register_shared: mr_vaddr 0x%p, mr_len %llu, "
4839 "mr_flags 0x%x, mr_as 0x%p, mr_exists %d, smrp 0x%p\n",
4840 (void *)(uintptr_t)mr_rp->mr_attr.mr_vaddr,
4841 (longlong_t)mr_rp->mr_attr.mr_len,
4842 mr_rp->mr_attr.mr_flags, mr_rp->mr_attr.mr_as,
4843 (int)(smrp->smr_mr_list != NULL), smrp);
4844
4845 /*
4846 * since we are in TRANSITIONING state, we are guaranteed
4847 * that we have exclusive access to smr_mr_list.
4848 */
4849 if (smrp->smr_mr_list != NULL) {
4850 ibt_smr_attr_t mem_sattr;
4851
4852 /*
4853 * a non-null smr_mr_list indicates that someone
4854 * else has already inserted an mr_resource into
4855 * smr_mr_list. we use the mr_handle from the first
4856 * element as an arg to ibt_register_shared_mr.
4857 */
4858 mem_sattr.mr_vaddr = smrp->smr_mr_list->mr_desc.md_vaddr;
4859 mem_sattr.mr_flags = mr_rp->mr_attr.mr_flags;
4860
4861 D2("mr_register_shared: mem_sattr vaddr 0x%p flags 0x%x\n",
4862 (void *)(uintptr_t)mem_sattr.mr_vaddr, mem_sattr.mr_flags);
4863 status = daplka_ibt_register_shared_mr(mr_rp, ia_rp->ia_hca_hdl,
4864 smrp->smr_mr_list->mr_hdl, mr_rp->mr_pd_res->pd_hdl,
4865 &mem_sattr, &mr_rp->mr_hdl, &mr_rp->mr_desc);
4866
4867 if (status != IBT_SUCCESS) {
4868 DERR("mr_register_shared: "
4869 "ibt_register_shared_mr error %d\n", status);
4870 *rvalp = (int)status;
4871 retval = 0;
4872 goto cleanup;
4873 }
4874 } else {
4875 /*
4876 * an mr does not exist yet. we need to create one
4877 * using ibt_register_mr.
4878 */
4879 status = daplka_ibt_register_mr(mr_rp, ia_rp->ia_hca_hdl,
4880 mr_rp->mr_pd_res->pd_hdl, &mr_rp->mr_attr,
4881 &mr_rp->mr_hdl, &mr_rp->mr_desc);
4882
4883 if (status != IBT_SUCCESS) {
4884 DERR("mr_register_shared: "
4885 "ibt_register_mr error %d\n", status);
4886 *rvalp = (int)status;
4887 retval = 0;
4888 goto cleanup;
4889 }
4890 }
4891
4892 mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
4893 mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
4894 mr_cb_data_in.mr_arg1 = (void *)mr_rp;
4895 mr_cb_data_in.mr_arg2 = NULL;
4896
4897 /* Pass the service driver mr cleanup handler to the hca driver */
4898 status = ibt_ci_data_in(ia_rp->ia_hca_hdl,
4899 IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
4900 &mr_cb_data_in, sizeof (mr_cb_data_in));
4901
4902 if (status != IBT_SUCCESS) {
4903 DERR("mr_register_shared: ibt_ci_data_in error(%d) ver(%d)",
4904 status, mr_cb_data_in.mr_rev);
4905 *rvalp = (int)status;
4906 retval = 0;
4907 goto cleanup;
4908 }
4909
4910 /*
4911 * we bump reference of mr_rp and enqueue it onto smrp.
4912 */
4913 DAPLKA_RS_REF(mr_rp);
4914 mr_rp->mr_next = smrp->smr_mr_list;
4915 smrp->smr_mr_list = mr_rp;
4916 mr_rp->mr_shared_mr = smrp;
4917
4918 /* insert into mr hash table */
4919 retval = daplka_hash_insert(&ia_rp->ia_mr_htbl,
4920 &mr_hkey, (void *)mr_rp);
4921 if (retval != 0) {
4922 DERR("mr_register_shared: cannot insert mr resource\n");
4923 goto cleanup;
4924 }
4925 inserted = B_TRUE;
4926 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mr_rp))
4927
4928 /*
4929 * at this point, there are two references to our mr resource.
4930 * one is kept in ia_mr_htbl. the other is kept in the list
4931 * within this shared mr object (smrp). when we deregister this
4932 * mr or when a callback invalidates this mr, the reference kept
4933 * by this shared mr object will be removed.
4934 */
4935
4936 args.mrs_lkey = mr_rp->mr_desc.md_lkey;
4937 args.mrs_rkey = mr_rp->mr_desc.md_rkey;
4938 args.mrs_hkey = mr_hkey;
4939
4940 retval = ddi_copyout((void *)&args, (void *)arg,
4941 sizeof (dapl_mr_register_shared_t), mode);
4942 if (retval != 0) {
4943 DERR("mr_register_shared: copyout error %d\n", retval);
4944 retval = EFAULT;
4945 goto cleanup;
4946 }
4947
4948 /*
4949 * set the state to READY to allow others to continue
4950 */
4951 mutex_enter(&daplka_shared_mr_lock);
4952 smrp->smr_state = DAPLKA_SMR_READY;
4953 cv_broadcast(&smrp->smr_cv);
4954 mutex_exit(&daplka_shared_mr_lock);
4955 return (0);
4956
4957 cleanup:;
4958 if (inserted) {
4959 daplka_mr_resource_t *free_rp = NULL;
4960
4961 (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
4962 (void **)&free_rp);
4963 if (free_rp != mr_rp) {
4964 DERR("mr_register_shared: "
4965 "cannot remove mr from hash table\n");
4966 /*
4967 * we can only get here if another thread
4968 * has completed the cleanup in mr_deregister
4969 */
4970 return (retval);
4971 }
4972 }
4973 if (smrp != NULL) {
4974 mutex_enter(&daplka_shared_mr_lock);
4975 ASSERT(smrp->smr_refcnt > 0);
4976 smrp->smr_refcnt--;
4977
4978 if (smrp->smr_refcnt == 0) {
4979 DERR("mr_register_shared: freeing smrp 0x%p\n", smrp);
4980 avl_remove(&daplka_shared_mr_tree, smrp);
4981 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*smrp))
4982 if (smrp->smr_mr_list != NULL) {
4983 /*
4984 * the refcnt is 0. if there is anything
4985 * left on the list, it must be ours.
4986 */
4987 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
4988 ASSERT(smrp->smr_mr_list == mr_rp);
4989 DAPLKA_RS_UNREF(mr_rp);
4990 smrp->smr_mr_list = NULL;
4991 ASSERT(mr_rp->mr_shared_mr == smrp);
4992 mr_rp->mr_shared_mr = NULL;
4993 ASSERT(mr_rp->mr_next == NULL);
4994 }
4995 smrp->smr_state = DAPLKA_SMR_FREED;
4996 cv_destroy(&smrp->smr_cv);
4997 kmem_free(smrp, sizeof (daplka_shared_mr_t));
4998 } else {
4999 DERR("mr_register_shared: resetting smr_state "
5000 "smrp 0x%p, %d waiters remain\n", smrp,
5001 smrp->smr_refcnt);
5002 ASSERT(smrp->smr_state == DAPLKA_SMR_TRANSITIONING);
5003 if (smrp->smr_mr_list != NULL && mr_rp != NULL) {
5004 daplka_mr_resource_t **mpp;
5005
5006 /*
5007 * search and remove mr_rp from smr_mr_list
5008 */
5009 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
5010 mpp = &smrp->smr_mr_list;
5011 while (*mpp != NULL) {
5012 if (*mpp == mr_rp) {
5013 *mpp = (*mpp)->mr_next;
5014 DAPLKA_RS_UNREF(mr_rp);
5015 ASSERT(mr_rp->mr_shared_mr ==
5016 smrp);
5017 mr_rp->mr_shared_mr = NULL;
5018 mr_rp->mr_next = NULL;
5019 break;
5020 }
5021 mpp = &(*mpp)->mr_next;
5022 }
5023 }
5024 /*
5025 * note that smr_state == READY does not necessarily
5026 * mean that smr_mr_list is non empty. for this case,
5027 * we are doing cleanup because of a failure. we set
5028 * the state to READY to allow other threads to
5029 * continue.
5030 */
5031 smrp->smr_state = DAPLKA_SMR_READY;
5032 cv_broadcast(&smrp->smr_cv);
5033 }
5034 mutex_exit(&daplka_shared_mr_lock);
5035 }
5036 if (mr_rp != NULL) {
5037 DAPLKA_RS_UNREF(mr_rp);
5038 }
5039 return (retval);
5040 }
5041
5042 /*
5043 * registers a memory region using the attributes of an
5044 * existing region.
5045 */
5046 /* ARGSUSED */
5047 static int
5048 daplka_mr_register_lmr(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5049 cred_t *cred, int *rvalp)
5050 {
5051 boolean_t inserted = B_FALSE;
5052 dapl_mr_register_lmr_t args;
5053 ibt_mr_data_in_t mr_cb_data_in;
5054 daplka_mr_resource_t *orig_mr_rp = NULL;
5055 daplka_mr_resource_t *mr_rp;
5056 ibt_smr_attr_t mem_sattr;
5057 uint64_t mr_hkey = 0;
5058 ibt_status_t status;
5059 int retval;
5060
5061 retval = ddi_copyin((void *)arg, &args,
5062 sizeof (dapl_mr_register_lmr_t), mode);
5063 if (retval != 0) {
5064 DERR("mr_register_lmr: copyin error %d\n", retval);
5065 return (EINVAL);
5066 }
5067 orig_mr_rp = (daplka_mr_resource_t *)
5068 daplka_hash_lookup(&ia_rp->ia_mr_htbl, args.mrl_orig_hkey);
5069 if (orig_mr_rp == NULL) {
5070 DERR("mr_register_lmr: cannot find mr resource\n");
5071 return (EINVAL);
5072 }
5073 ASSERT(DAPLKA_RS_TYPE(orig_mr_rp) == DAPL_TYPE_MR);
5074
5075 mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
5076 if (mr_rp == NULL) {
5077 DERR("mr_register_lmr: cannot allocate mr resource\n");
5078 retval = ENOMEM;
5079 goto cleanup;
5080 }
5081 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
5082 DAPLKA_RS_INIT(mr_rp, DAPL_TYPE_MR,
5083 DAPLKA_RS_RNUM(ia_rp), daplka_mr_destroy);
5084
5085 mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
5086 mr_rp->mr_hca = ia_rp->ia_hca;
5087 mr_rp->mr_hca_hdl = ia_rp->ia_hca_hdl;
5088 mr_rp->mr_next = NULL;
5089 mr_rp->mr_shared_mr = NULL;
5090
5091 DAPLKA_RS_REF(orig_mr_rp->mr_pd_res);
5092 mr_rp->mr_pd_res = orig_mr_rp->mr_pd_res;
5093 mr_rp->mr_attr = orig_mr_rp->mr_attr;
5094
5095 /* Pass the IO addr that was returned while allocating the orig MR */
5096 mem_sattr.mr_vaddr = orig_mr_rp->mr_desc.md_vaddr;
5097 mem_sattr.mr_flags = args.mrl_flags | IBT_MR_NOSLEEP;
5098
5099 status = daplka_ibt_register_shared_mr(mr_rp, ia_rp->ia_hca_hdl,
5100 orig_mr_rp->mr_hdl, mr_rp->mr_pd_res->pd_hdl, &mem_sattr,
5101 &mr_rp->mr_hdl, &mr_rp->mr_desc);
5102
5103 if (status != IBT_SUCCESS) {
5104 DERR("mr_register_lmr: ibt_register_shared_mr error %d\n",
5105 status);
5106 *rvalp = (int)status;
5107 retval = 0;
5108 goto cleanup;
5109 }
5110
5111 mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
5112 mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
5113 mr_cb_data_in.mr_arg1 = (void *)mr_rp;
5114 mr_cb_data_in.mr_arg2 = NULL;
5115
5116 /* Pass the service driver mr cleanup handler to the hca driver */
5117 status = ibt_ci_data_in(ia_rp->ia_hca_hdl,
5118 IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
5119 &mr_cb_data_in, sizeof (mr_cb_data_in));
5120
5121 if (status != IBT_SUCCESS) {
5122 DERR("mr_register_lmr: ibt_ci_data_in error(%d) ver(%d)",
5123 status, mr_cb_data_in.mr_rev);
5124 *rvalp = (int)status;
5125 retval = 0;
5126 goto cleanup;
5127 }
5128 mr_rp->mr_attr.mr_len = orig_mr_rp->mr_attr.mr_len;
5129 mr_rp->mr_attr.mr_flags = mem_sattr.mr_flags;
5130
5131 /* insert into mr hash table */
5132 retval = daplka_hash_insert(&ia_rp->ia_mr_htbl, &mr_hkey,
5133 (void *)mr_rp);
5134 if (retval != 0) {
5135 DERR("mr_register: cannot insert mr resource into mr_htbl\n");
5136 goto cleanup;
5137 }
5138 inserted = B_TRUE;
5139 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mr_rp))
5140
5141 args.mrl_lkey = mr_rp->mr_desc.md_lkey;
5142 args.mrl_rkey = mr_rp->mr_desc.md_rkey;
5143 args.mrl_hkey = mr_hkey;
5144
5145 retval = ddi_copyout((void *)&args, (void *)arg,
5146 sizeof (dapl_mr_register_lmr_t), mode);
5147 if (retval != 0) {
5148 DERR("mr_register_lmr: copyout error %d\n", retval);
5149 retval = EFAULT;
5150 goto cleanup;
5151 }
5152 if (orig_mr_rp != NULL) {
5153 DAPLKA_RS_UNREF(orig_mr_rp);
5154 }
5155 return (0);
5156
5157 cleanup:;
5158 if (inserted) {
5159 daplka_mr_resource_t *free_rp = NULL;
5160
5161 (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
5162 (void **)&free_rp);
5163 if (free_rp != mr_rp) {
5164 DERR("mr_register: cannot remove mr from hash table\n");
5165 /*
5166 * we can only get here if another thread
5167 * has completed the cleanup in mr_deregister
5168 */
5169 return (retval);
5170 }
5171 }
5172 if (orig_mr_rp != NULL) {
5173 DAPLKA_RS_UNREF(orig_mr_rp);
5174 }
5175 if (mr_rp != NULL) {
5176 DAPLKA_RS_UNREF(mr_rp);
5177 }
5178 return (retval);
5179 }
5180
5181 /*
5182 * this function is called by mr_deregister and mr_cleanup_callback to
5183 * remove a mr resource from the shared mr object mr_rp->mr_shared_mr.
5184 * if mr_shared_mr is already NULL, that means the region being
5185 * deregistered or invalidated is not a shared mr region and we can
5186 * return immediately.
5187 */
5188 static void
5189 daplka_shared_mr_free(daplka_mr_resource_t *mr_rp)
5190 {
5191 daplka_shared_mr_t *smrp;
5192
5193 /*
5194 * we need a lock because mr_callback also checks this field.
5195 * for the rare case that mr_deregister and mr_cleanup_callback
5196 * gets called simultaneously, we are guaranteed that smrp won't
5197 * be dereferenced twice because either function will find
5198 * mr_shared_mr to be NULL.
5199 */
5200 mutex_enter(&mr_rp->mr_lock);
5201 smrp = mr_rp->mr_shared_mr;
5202 mr_rp->mr_shared_mr = NULL;
5203 mutex_exit(&mr_rp->mr_lock);
5204
5205 if (smrp != NULL) {
5206 daplka_mr_resource_t **mpp;
5207 boolean_t mr_found = B_FALSE;
5208
5209 mutex_enter(&daplka_shared_mr_lock);
5210 ASSERT(smrp->smr_refcnt > 0);
5211 while (smrp->smr_state == DAPLKA_SMR_TRANSITIONING) {
5212 cv_wait(&smrp->smr_cv, &daplka_shared_mr_lock);
5213 }
5214 ASSERT(smrp->smr_state == DAPLKA_SMR_READY);
5215 smrp->smr_state = DAPLKA_SMR_TRANSITIONING;
5216 smrp->smr_refcnt--;
5217
5218 /*
5219 * search and remove mr_rp from smr_mr_list.
5220 * also UNREF mr_rp because it is no longer
5221 * on the list.
5222 */
5223 mpp = &smrp->smr_mr_list;
5224 while (*mpp != NULL) {
5225 if (*mpp == mr_rp) {
5226 *mpp = (*mpp)->mr_next;
5227 DAPLKA_RS_UNREF(mr_rp);
5228 mr_rp->mr_next = NULL;
5229 mr_found = B_TRUE;
5230 break;
5231 }
5232 mpp = &(*mpp)->mr_next;
5233 }
5234 /*
5235 * since mr_clean_callback may not touch smr_mr_list
5236 * at this time (due to smr_state), we can be sure
5237 * that we can find and remove mr_rp from smr_mr_list
5238 */
5239 ASSERT(mr_found);
5240 if (smrp->smr_refcnt == 0) {
5241 D3("shared_mr_free: freeing smrp 0x%p\n", smrp);
5242 avl_remove(&daplka_shared_mr_tree, smrp);
5243 ASSERT(smrp->smr_mr_list == NULL);
5244 smrp->smr_state = DAPLKA_SMR_FREED;
5245 cv_destroy(&smrp->smr_cv);
5246 kmem_free(smrp, sizeof (daplka_shared_mr_t));
5247 } else {
5248 D3("shared_mr_free: smrp 0x%p, refcnt %d\n",
5249 smrp, smrp->smr_refcnt);
5250 smrp->smr_state = DAPLKA_SMR_READY;
5251 cv_broadcast(&smrp->smr_cv);
5252 }
5253 mutex_exit(&daplka_shared_mr_lock);
5254 }
5255 }
5256
5257 /*
5258 * deregisters a memory region.
5259 * if mr is shared, remove reference from global shared mr object.
5260 * release the initial reference to the mr. if the mr's refcnt is
5261 * zero, call mr_destroy to free mr.
5262 */
5263 /* ARGSUSED */
5264 static int
5265 daplka_mr_deregister(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5266 cred_t *cred, int *rvalp)
5267 {
5268 daplka_mr_resource_t *mr_rp;
5269 dapl_mr_deregister_t args;
5270 int retval;
5271
5272 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_deregister_t),
5273 mode);
5274 if (retval != 0) {
5275 DERR("mr_deregister: copyin error %d\n", retval);
5276 return (EINVAL);
5277 }
5278 retval = daplka_hash_remove(&ia_rp->ia_mr_htbl,
5279 args.mrd_hkey, (void **)&mr_rp);
5280 if (retval != 0 || mr_rp == NULL) {
5281 DERR("mr_deregister: cannot find mr resource\n");
5282 return (EINVAL);
5283 }
5284 ASSERT(DAPLKA_RS_TYPE(mr_rp) == DAPL_TYPE_MR);
5285
5286 daplka_shared_mr_free(mr_rp);
5287 DAPLKA_RS_UNREF(mr_rp);
5288 return (0);
5289 }
5290
5291 /*
5292 * sync local memory regions on RDMA read or write.
5293 */
5294 /* ARGSUSED */
5295 static int
5296 daplka_mr_sync(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5297 cred_t *cred, int *rvalp)
5298 {
5299 dapl_mr_sync_t args;
5300 daplka_mr_resource_t *mr_rp[DAPL_MR_PER_SYNC];
5301 ibt_mr_sync_t mrs[DAPL_MR_PER_SYNC];
5302 uint32_t sync_direction_flags;
5303 ibt_status_t status;
5304 int i, j;
5305 int retval;
5306
5307 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_sync_t), mode);
5308 if (retval != 0) {
5309 DERR("mr_sync: copyin error %d\n", retval);
5310 return (EFAULT);
5311 }
5312
5313 /* number of segments bound check */
5314 if (args.mrs_numseg > DAPL_MR_PER_SYNC) {
5315 DERR("mr_sync: number of segments too large\n");
5316 return (EINVAL);
5317 }
5318
5319 /* translate MR sync direction flag */
5320 if (args.mrs_flags == DAPL_MR_SYNC_RDMA_RD) {
5321 sync_direction_flags = IBT_SYNC_READ;
5322 } else if (args.mrs_flags == DAPL_MR_SYNC_RDMA_WR) {
5323 sync_direction_flags = IBT_SYNC_WRITE;
5324 } else {
5325 DERR("mr_sync: unknown flags\n");
5326 return (EINVAL);
5327 }
5328
5329 /*
5330 * all the segments are going to be sync'd by ibtl together
5331 */
5332 for (i = 0; i < args.mrs_numseg; i++) {
5333 mr_rp[i] = (daplka_mr_resource_t *)daplka_hash_lookup(
5334 &ia_rp->ia_mr_htbl, args.mrs_vec[i].mrsv_hkey);
5335 if (mr_rp[i] == NULL) {
5336 for (j = 0; j < i; j++) {
5337 DAPLKA_RS_UNREF(mr_rp[j]);
5338 }
5339 DERR("mr_sync: lookup error\n");
5340 return (EINVAL);
5341 }
5342 ASSERT(DAPLKA_RS_TYPE(mr_rp[i]) == DAPL_TYPE_MR);
5343 mrs[i].ms_handle = mr_rp[i]->mr_hdl;
5344 mrs[i].ms_vaddr = args.mrs_vec[i].mrsv_va;
5345 mrs[i].ms_len = args.mrs_vec[i].mrsv_len;
5346 mrs[i].ms_flags = sync_direction_flags;
5347 }
5348
5349 status = ibt_sync_mr(ia_rp->ia_hca_hdl, mrs, args.mrs_numseg);
5350 if (status != IBT_SUCCESS) {
5351 DERR("mr_sync: ibt_sync_mr error %d\n", status);
5352 *rvalp = (int)status;
5353 }
5354 for (i = 0; i < args.mrs_numseg; i++) {
5355 DAPLKA_RS_UNREF(mr_rp[i]);
5356 }
5357 return (0);
5358 }
5359
5360 /*
5361 * destroys a memory region.
5362 * called when refcnt drops to zero.
5363 */
5364 static int
5365 daplka_mr_destroy(daplka_resource_t *gen_rp)
5366 {
5367 daplka_mr_resource_t *mr_rp = (daplka_mr_resource_t *)gen_rp;
5368 ibt_status_t status;
5369
5370 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_rp))
5371 ASSERT(DAPLKA_RS_REFCNT(mr_rp) == 0);
5372 ASSERT(mr_rp->mr_shared_mr == NULL);
5373 D3("mr_destroy: entering, mr_rp 0x%p, rnum %d\n",
5374 mr_rp, DAPLKA_RS_RNUM(mr_rp));
5375
5376 /*
5377 * deregister mr
5378 */
5379 if (mr_rp->mr_hdl) {
5380 status = daplka_ibt_deregister_mr(mr_rp, mr_rp->mr_hca_hdl,
5381 mr_rp->mr_hdl);
5382 if (status != IBT_SUCCESS) {
5383 DERR("mr_destroy: ibt_deregister_mr returned %d\n",
5384 status);
5385 }
5386 mr_rp->mr_hdl = NULL;
5387 D3("mr_destroy: mr deregistered\n");
5388 }
5389 mr_rp->mr_attr.mr_vaddr = 0;
5390
5391 /*
5392 * release reference on PD
5393 */
5394 if (mr_rp->mr_pd_res != NULL) {
5395 DAPLKA_RS_UNREF(mr_rp->mr_pd_res);
5396 mr_rp->mr_pd_res = NULL;
5397 }
5398 mutex_destroy(&mr_rp->mr_lock);
5399 DAPLKA_RS_FINI(mr_rp);
5400 kmem_free(mr_rp, sizeof (daplka_mr_resource_t));
5401 D3("mr_destroy: exiting, mr_rp 0x%p\n", mr_rp);
5402 return (0);
5403 }
5404
5405 /*
5406 * this function is called by daplka_hash_destroy for
5407 * freeing MR resource objects
5408 */
5409 static void
5410 daplka_hash_mr_free(void *obj)
5411 {
5412 daplka_mr_resource_t *mr_rp = (daplka_mr_resource_t *)obj;
5413
5414 daplka_shared_mr_free(mr_rp);
5415 DAPLKA_RS_UNREF(mr_rp);
5416 }
5417
5418 /*
5419 * comparison function used for finding a shared mr object
5420 * from the global shared mr avl tree.
5421 */
5422 static int
5423 daplka_shared_mr_cmp(const void *smr1, const void *smr2)
5424 {
5425 daplka_shared_mr_t *s1 = (daplka_shared_mr_t *)smr1;
5426 daplka_shared_mr_t *s2 = (daplka_shared_mr_t *)smr2;
5427 int i;
5428
5429 for (i = 4; i >= 0; i--) {
5430 if (s1->smr_cookie.mc_uint_arr[i] <
5431 s2->smr_cookie.mc_uint_arr[i]) {
5432 return (-1);
5433 }
5434 if (s1->smr_cookie.mc_uint_arr[i] >
5435 s2->smr_cookie.mc_uint_arr[i]) {
5436 return (1);
5437 }
5438 }
5439 return (0);
5440 }
5441
5442 /*
5443 * allocates a protection domain.
5444 */
5445 /* ARGSUSED */
5446 static int
5447 daplka_pd_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5448 cred_t *cred, int *rvalp)
5449 {
5450 dapl_pd_alloc_t args;
5451 daplka_pd_resource_t *pd_rp;
5452 ibt_status_t status;
5453 uint64_t pd_hkey = 0;
5454 boolean_t inserted = B_FALSE;
5455 int retval;
5456
5457 pd_rp = kmem_zalloc(sizeof (*pd_rp), daplka_km_flags);
5458 if (pd_rp == NULL) {
5459 DERR("pd_alloc: cannot allocate pd resource\n");
5460 return (ENOMEM);
5461 }
5462 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd_rp))
5463 DAPLKA_RS_INIT(pd_rp, DAPL_TYPE_PD,
5464 DAPLKA_RS_RNUM(ia_rp), daplka_pd_destroy);
5465
5466 pd_rp->pd_hca = ia_rp->ia_hca;
5467 pd_rp->pd_hca_hdl = ia_rp->ia_hca_hdl;
5468 status = daplka_ibt_alloc_pd(pd_rp, pd_rp->pd_hca_hdl,
5469 IBT_PD_NO_FLAGS, &pd_rp->pd_hdl);
5470 if (status != IBT_SUCCESS) {
5471 DERR("pd_alloc: ibt_alloc_pd returned %d\n", status);
5472 *rvalp = (int)status;
5473 retval = 0;
5474 goto cleanup;
5475 }
5476
5477 /* insert into pd hash table */
5478 retval = daplka_hash_insert(&ia_rp->ia_pd_htbl,
5479 &pd_hkey, (void *)pd_rp);
5480 if (retval != 0) {
5481 DERR("pd_alloc: cannot insert pd resource into pd_htbl\n");
5482 goto cleanup;
5483 }
5484 inserted = B_TRUE;
5485 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pd_rp))
5486
5487 /* return hkey to library */
5488 args.pda_hkey = pd_hkey;
5489
5490 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_pd_alloc_t),
5491 mode);
5492 if (retval != 0) {
5493 DERR("pd_alloc: copyout error %d\n", retval);
5494 retval = EFAULT;
5495 goto cleanup;
5496 }
5497 return (0);
5498
5499 cleanup:;
5500 if (inserted) {
5501 daplka_pd_resource_t *free_rp = NULL;
5502
5503 (void) daplka_hash_remove(&ia_rp->ia_pd_htbl, pd_hkey,
5504 (void **)&free_rp);
5505 if (free_rp != pd_rp) {
5506 DERR("pd_alloc: cannot remove pd from hash table\n");
5507 /*
5508 * we can only get here if another thread
5509 * has completed the cleanup in pd_free
5510 */
5511 return (retval);
5512 }
5513 }
5514 DAPLKA_RS_UNREF(pd_rp);
5515 return (retval);
5516 }
5517
5518 /*
5519 * destroys a protection domain.
5520 * called when refcnt drops to zero.
5521 */
5522 static int
5523 daplka_pd_destroy(daplka_resource_t *gen_rp)
5524 {
5525 daplka_pd_resource_t *pd_rp = (daplka_pd_resource_t *)gen_rp;
5526 ibt_status_t status;
5527
5528 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd_rp))
5529 ASSERT(DAPLKA_RS_REFCNT(pd_rp) == 0);
5530 D3("pd_destroy: entering, pd_rp %p, rnum %d\n",
5531 pd_rp, DAPLKA_RS_RNUM(pd_rp));
5532
5533 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
5534 if (pd_rp->pd_hdl != NULL) {
5535 status = daplka_ibt_free_pd(pd_rp, pd_rp->pd_hca_hdl,
5536 pd_rp->pd_hdl);
5537 if (status != IBT_SUCCESS) {
5538 DERR("pd_destroy: ibt_free_pd returned %d\n", status);
5539 }
5540 }
5541 DAPLKA_RS_FINI(pd_rp);
5542 kmem_free(pd_rp, sizeof (daplka_pd_resource_t));
5543 D3("pd_destroy: exiting, pd_rp %p\n", pd_rp);
5544 return (0);
5545 }
5546
5547 static void
5548 daplka_hash_pd_free(void *obj)
5549 {
5550 daplka_pd_resource_t *pd_rp = (daplka_pd_resource_t *)obj;
5551
5552 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
5553 DAPLKA_RS_UNREF(pd_rp);
5554 }
5555
5556 /*
5557 * removes the pd reference from ia_pd_htbl and releases the
5558 * initial reference to the pd. also destroys the pd if the refcnt
5559 * is zero.
5560 */
5561 /* ARGSUSED */
5562 static int
5563 daplka_pd_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5564 cred_t *cred, int *rvalp)
5565 {
5566 daplka_pd_resource_t *pd_rp;
5567 dapl_pd_free_t args;
5568 int retval;
5569
5570 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_pd_free_t), mode);
5571 if (retval != 0) {
5572 DERR("pd_free: copyin error %d\n", retval);
5573 return (EINVAL);
5574 }
5575
5576 retval = daplka_hash_remove(&ia_rp->ia_pd_htbl,
5577 args.pdf_hkey, (void **)&pd_rp);
5578 if (retval != 0 || pd_rp == NULL) {
5579 DERR("pd_free: cannot find pd resource\n");
5580 return (EINVAL);
5581 }
5582 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
5583
5584 /* UNREF calls the actual free function when refcnt is zero */
5585 DAPLKA_RS_UNREF(pd_rp);
5586 return (0);
5587 }
5588
5589 /*
5590 * allocates a memory window
5591 */
5592 /* ARGSUSED */
5593 static int
5594 daplka_mw_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5595 cred_t *cred, int *rvalp)
5596 {
5597 daplka_pd_resource_t *pd_rp;
5598 daplka_mw_resource_t *mw_rp;
5599 dapl_mw_alloc_t args;
5600 ibt_status_t status;
5601 boolean_t inserted = B_FALSE;
5602 uint64_t mw_hkey;
5603 ibt_rkey_t mw_rkey;
5604 int retval;
5605
5606 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mw_alloc_t), mode);
5607 if (retval != 0) {
5608 DERR("mw_alloc: copyin error %d\n", retval);
5609 return (EFAULT);
5610 }
5611
5612 /*
5613 * Allocate and initialize a MW resource
5614 */
5615 mw_rp = kmem_zalloc(sizeof (daplka_mw_resource_t), daplka_km_flags);
5616 if (mw_rp == NULL) {
5617 DERR("mw_alloc: cannot allocate mw resource\n");
5618 return (ENOMEM);
5619 }
5620 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw_rp))
5621 DAPLKA_RS_INIT(mw_rp, DAPL_TYPE_MW,
5622 DAPLKA_RS_RNUM(ia_rp), daplka_mw_destroy);
5623
5624 mutex_init(&mw_rp->mw_lock, NULL, MUTEX_DRIVER, NULL);
5625 mw_rp->mw_hca = ia_rp->ia_hca;
5626 mw_rp->mw_hca_hdl = ia_rp->ia_hca_hdl;
5627
5628 /* get pd handle */
5629 pd_rp = (daplka_pd_resource_t *)
5630 daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mw_pd_hkey);
5631 if (pd_rp == NULL) {
5632 DERR("mw_alloc: cannot find pd resource\n");
5633 goto cleanup;
5634 }
5635 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
5636
5637 mw_rp->mw_pd_res = pd_rp;
5638
5639 status = daplka_ibt_alloc_mw(mw_rp, mw_rp->mw_hca_hdl,
5640 pd_rp->pd_hdl, IBT_MW_NOSLEEP, &mw_rp->mw_hdl, &mw_rkey);
5641
5642 if (status != IBT_SUCCESS) {
5643 DERR("mw_alloc: ibt_alloc_mw returned %d\n", status);
5644 *rvalp = (int)status;
5645 retval = 0;
5646 goto cleanup;
5647 }
5648
5649 mutex_enter(&ia_rp->ia_lock);
5650 switch (ia_rp->ia_state) {
5651 case DAPLKA_IA_INIT:
5652 ia_rp->ia_state = DAPLKA_IA_MW_ALLOC_IN_PROGRESS;
5653 ia_rp->ia_mw_alloccnt++;
5654 retval = 0;
5655 break;
5656 case DAPLKA_IA_MW_ALLOC_IN_PROGRESS:
5657 /* another mw_alloc is already in progress increase cnt */
5658 ia_rp->ia_mw_alloccnt++;
5659 retval = 0;
5660 break;
5661 case DAPLKA_IA_MW_FREEZE_IN_PROGRESS:
5662 /* FALLTHRU */
5663 case DAPLKA_IA_MW_FROZEN:
5664 /*
5665 * IA is being or already frozen don't allow more MWs to be
5666 * allocated.
5667 */
5668 DERR("mw_alloc: IA is freezing MWs (state=%d)\n",
5669 ia_rp->ia_state);
5670 retval = EINVAL;
5671 break;
5672 default:
5673 ASSERT(!"Invalid IA state in mw_alloc");
5674 DERR("mw_alloc: IA state=%d invalid\n", ia_rp->ia_state);
5675 retval = EINVAL;
5676 break;
5677 }
5678 mutex_exit(&ia_rp->ia_lock);
5679 /* retval is 0 when ia_mw_alloccnt is incremented */
5680 if (retval != 0) {
5681 goto cleanup;
5682 }
5683
5684 /* insert into mw hash table */
5685 mw_hkey = 0;
5686 retval = daplka_hash_insert(&ia_rp->ia_mw_htbl, &mw_hkey,
5687 (void *)mw_rp);
5688 if (retval != 0) {
5689 DERR("mw_alloc: cannot insert mw resource into mw_htbl\n");
5690 mutex_enter(&ia_rp->ia_lock);
5691 ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS);
5692 ia_rp->ia_mw_alloccnt--;
5693 if (ia_rp->ia_mw_alloccnt == 0) {
5694 ia_rp->ia_state = DAPLKA_IA_INIT;
5695 cv_broadcast(&ia_rp->ia_cv);
5696 }
5697 mutex_exit(&ia_rp->ia_lock);
5698 goto cleanup;
5699 }
5700 inserted = B_TRUE;
5701 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mw_rp))
5702
5703 D3("mw_alloc: ibt_alloc_mw mw_hdl(%p) mw_rkey(0x%llx)\n",
5704 mw_rp->mw_hdl, (longlong_t)mw_rkey);
5705
5706 mutex_enter(&ia_rp->ia_lock);
5707 /*
5708 * We are done with mw_alloc if this was the last mw_alloc
5709 * change state back to DAPLKA_IA_INIT and wake up waiters
5710 * specifically the unlock callback.
5711 */
5712 ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS);
5713 ia_rp->ia_mw_alloccnt--;
5714 if (ia_rp->ia_mw_alloccnt == 0) {
5715 ia_rp->ia_state = DAPLKA_IA_INIT;
5716 cv_broadcast(&ia_rp->ia_cv);
5717 }
5718 mutex_exit(&ia_rp->ia_lock);
5719
5720 args.mw_hkey = mw_hkey;
5721 args.mw_rkey = mw_rkey;
5722
5723 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_mw_alloc_t),
5724 mode);
5725 if (retval != 0) {
5726 DERR("mw_alloc: copyout error %d\n", retval);
5727 retval = EFAULT;
5728 goto cleanup;
5729 }
5730 return (0);
5731
5732 cleanup:;
5733 if (inserted) {
5734 daplka_mw_resource_t *free_rp = NULL;
5735
5736 (void) daplka_hash_remove(&ia_rp->ia_mw_htbl, mw_hkey,
5737 (void **)&free_rp);
5738 if (free_rp != mw_rp) {
5739 DERR("mw_alloc: cannot remove mw from hash table\n");
5740 /*
5741 * we can only get here if another thread
5742 * has completed the cleanup in mw_free
5743 */
5744 return (retval);
5745 }
5746 }
5747 DAPLKA_RS_UNREF(mw_rp);
5748 return (retval);
5749 }
5750
5751 /*
5752 * removes the mw reference from ia_mw_htbl and releases the
5753 * initial reference to the mw. also destroys the mw if the refcnt
5754 * is zero.
5755 */
5756 /* ARGSUSED */
5757 static int
5758 daplka_mw_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5759 cred_t *cred, int *rvalp)
5760 {
5761 daplka_mw_resource_t *mw_rp = NULL;
5762 dapl_mw_free_t args;
5763 int retval = 0;
5764
5765 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mw_free_t), mode);
5766 if (retval != 0) {
5767 DERR("mw_free: copyin error %d\n", retval);
5768 return (EFAULT);
5769 }
5770
5771 retval = daplka_hash_remove(&ia_rp->ia_mw_htbl, args.mw_hkey,
5772 (void **)&mw_rp);
5773 if (retval != 0 || mw_rp == NULL) {
5774 DERR("mw_free: cannot find mw resrc (0x%llx)\n",
5775 (longlong_t)args.mw_hkey);
5776 return (EINVAL);
5777 }
5778
5779 ASSERT(DAPLKA_RS_TYPE(mw_rp) == DAPL_TYPE_MW);
5780
5781 /* UNREF calls the actual free function when refcnt is zero */
5782 DAPLKA_RS_UNREF(mw_rp);
5783 return (retval);
5784 }
5785
5786 /*
5787 * destroys the memory window.
5788 * called when refcnt drops to zero.
5789 */
5790 static int
5791 daplka_mw_destroy(daplka_resource_t *gen_rp)
5792 {
5793 daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)gen_rp;
5794 ibt_status_t status;
5795
5796 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw_rp))
5797 ASSERT(DAPLKA_RS_REFCNT(mw_rp) == 0);
5798 D3("mw_destroy: entering, mw_rp 0x%p, rnum %d\n",
5799 mw_rp, DAPLKA_RS_RNUM(mw_rp));
5800
5801 /*
5802 * free memory window
5803 */
5804 if (mw_rp->mw_hdl) {
5805 status = daplka_ibt_free_mw(mw_rp, mw_rp->mw_hca_hdl,
5806 mw_rp->mw_hdl);
5807 if (status != IBT_SUCCESS) {
5808 DERR("mw_destroy: ibt_free_mw returned %d\n", status);
5809 }
5810 mw_rp->mw_hdl = NULL;
5811 D3("mw_destroy: mw freed\n");
5812 }
5813
5814 /*
5815 * release reference on PD
5816 */
5817 if (mw_rp->mw_pd_res != NULL) {
5818 DAPLKA_RS_UNREF(mw_rp->mw_pd_res);
5819 mw_rp->mw_pd_res = NULL;
5820 }
5821 mutex_destroy(&mw_rp->mw_lock);
5822 DAPLKA_RS_FINI(mw_rp);
5823 kmem_free(mw_rp, sizeof (daplka_mw_resource_t));
5824 D3("mw_destroy: exiting, mw_rp 0x%p\n", mw_rp);
5825 return (0);
5826 }
5827
5828 static void
5829 daplka_hash_mw_free(void *obj)
5830 {
5831 daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)obj;
5832
5833 ASSERT(DAPLKA_RS_TYPE(mw_rp) == DAPL_TYPE_MW);
5834 DAPLKA_RS_UNREF(mw_rp);
5835 }
5836
5837 /*
5838 * SRQ ioctls and supporting functions
5839 */
5840 /* ARGSUSED */
5841 static int
5842 daplka_srq_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5843 cred_t *cred, int *rvalp)
5844 {
5845 daplka_srq_resource_t *srq_rp;
5846 daplka_pd_resource_t *pd_rp;
5847 dapl_srq_create_t args;
5848 ibt_srq_sizes_t srq_sizes;
5849 ibt_srq_sizes_t srq_real_sizes;
5850 ibt_hca_attr_t *hca_attrp;
5851 uint64_t srq_hkey = 0;
5852 boolean_t inserted = B_FALSE;
5853 int retval;
5854 ibt_status_t status;
5855
5856 D3("srq_create: enter\n");
5857 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_create_t),
5858 mode);
5859 if (retval != 0) {
5860 DERR("srq_create: copyin error %d\n", retval);
5861 return (EFAULT);
5862 }
5863 srq_rp = kmem_zalloc(sizeof (daplka_srq_resource_t), daplka_km_flags);
5864 if (srq_rp == NULL) {
5865 DERR("srq_create: cannot allocate ep_rp\n");
5866 return (ENOMEM);
5867 }
5868 DAPLKA_RS_INIT(srq_rp, DAPL_TYPE_SRQ,
5869 DAPLKA_RS_RNUM(ia_rp), daplka_srq_destroy);
5870
5871 srq_rp->srq_hca = ia_rp->ia_hca;
5872 srq_rp->srq_hca_hdl = ia_rp->ia_hca_hdl;
5873 mutex_init(&srq_rp->srq_lock, NULL, MUTEX_DRIVER, NULL);
5874
5875 /* get pd handle */
5876 pd_rp = (daplka_pd_resource_t *)
5877 daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.srqc_pd_hkey);
5878 if (pd_rp == NULL) {
5879 DERR("srq_create: cannot find pd resource\n");
5880 retval = EINVAL;
5881 goto cleanup;
5882 }
5883 ASSERT(DAPLKA_RS_TYPE(pd_rp) == DAPL_TYPE_PD);
5884 srq_rp->srq_pd_res = pd_rp;
5885
5886 /*
5887 * these checks ensure that the requested SRQ sizes
5888 * are within the limits supported by the chosen HCA.
5889 */
5890 hca_attrp = &ia_rp->ia_hca->hca_attr;
5891 if (args.srqc_sizes.srqs_sz > hca_attrp->hca_max_srqs_sz) {
5892 DERR("srq_create: invalid srqs_sz %d\n",
5893 args.srqc_sizes.srqs_sz);
5894 retval = EINVAL;
5895 goto cleanup;
5896 }
5897 if (args.srqc_sizes.srqs_sgl > hca_attrp->hca_max_srq_sgl) {
5898 DERR("srq_create: invalid srqs_sgl %d\n",
5899 args.srqc_sizes.srqs_sgl);
5900 retval = EINVAL;
5901 goto cleanup;
5902 }
5903
5904 D3("srq_create: srq_sgl %d, srq_sz %d\n",
5905 args.srqc_sizes.srqs_sgl, args.srqc_sizes.srqs_sz);
5906
5907 srq_sizes.srq_wr_sz = args.srqc_sizes.srqs_sz;
5908 srq_sizes.srq_sgl_sz = args.srqc_sizes.srqs_sgl;
5909
5910 /* create srq */
5911 status = daplka_ibt_alloc_srq(srq_rp, ia_rp->ia_hca_hdl,
5912 IBT_SRQ_USER_MAP, pd_rp->pd_hdl, &srq_sizes, &srq_rp->srq_hdl,
5913 &srq_real_sizes);
5914 if (status != IBT_SUCCESS) {
5915 DERR("srq_create: alloc_srq returned %d\n", status);
5916 *rvalp = (int)status;
5917 retval = 0;
5918 goto cleanup;
5919 }
5920
5921 args.srqc_real_sizes.srqs_sz = srq_real_sizes.srq_wr_sz;
5922 args.srqc_real_sizes.srqs_sgl = srq_real_sizes.srq_sgl_sz;
5923
5924 /* Get HCA-specific data_out info */
5925 status = ibt_ci_data_out(ia_rp->ia_hca_hdl,
5926 IBT_CI_NO_FLAGS, IBT_HDL_SRQ, (void *)srq_rp->srq_hdl,
5927 &args.srqc_data_out, sizeof (args.srqc_data_out));
5928
5929 if (status != IBT_SUCCESS) {
5930 DERR("srq_create: ibt_ci_data_out error(%d)\n", status);
5931 *rvalp = (int)status;
5932 retval = 0;
5933 goto cleanup;
5934 }
5935
5936 srq_rp->srq_real_size = srq_real_sizes.srq_wr_sz;
5937
5938 /* preparing to copyout map_data back to the library */
5939 args.srqc_real_sizes.srqs_sz = srq_real_sizes.srq_wr_sz;
5940 args.srqc_real_sizes.srqs_sgl = srq_real_sizes.srq_sgl_sz;
5941
5942 /* insert into srq hash table */
5943 retval = daplka_hash_insert(&ia_rp->ia_srq_htbl,
5944 &srq_hkey, (void *)srq_rp);
5945 if (retval != 0) {
5946 DERR("srq_create: cannot insert srq resource into srq_htbl\n");
5947 goto cleanup;
5948 }
5949 inserted = B_TRUE;
5950
5951 /* return hkey to library */
5952 args.srqc_hkey = srq_hkey;
5953
5954 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_srq_create_t),
5955 mode);
5956 if (retval != 0) {
5957 DERR("srq_create: copyout error %d\n", retval);
5958 retval = EFAULT;
5959 goto cleanup;
5960 }
5961
5962 D3("srq_create: %p, 0x%llx\n", srq_rp->srq_hdl, (longlong_t)srq_hkey);
5963 D3(" sz(%d) sgl(%d)\n",
5964 args.srqc_real_sizes.srqs_sz, args.srqc_real_sizes.srqs_sgl);
5965 D3("srq_create: exit\n");
5966 return (0);
5967
5968 cleanup:
5969 if (inserted) {
5970 daplka_srq_resource_t *free_rp = NULL;
5971
5972 (void) daplka_hash_remove(&ia_rp->ia_srq_htbl, srq_hkey,
5973 (void **)&free_rp);
5974 if (free_rp != srq_rp) {
5975 /*
5976 * this case is impossible because ep_free will
5977 * wait until our state transition is complete.
5978 */
5979 DERR("srq_create: cannot remove srq from hash table\n");
5980 ASSERT(B_FALSE);
5981 return (retval);
5982 }
5983 }
5984 DAPLKA_RS_UNREF(srq_rp);
5985 return (retval);
5986 }
5987
5988 /*
5989 * Resize an existing SRQ
5990 */
5991 /* ARGSUSED */
5992 static int
5993 daplka_srq_resize(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
5994 cred_t *cred, int *rvalp)
5995 {
5996 daplka_srq_resource_t *srq_rp = NULL;
5997 ibt_hca_attr_t *hca_attrp;
5998 dapl_srq_resize_t args;
5999 ibt_status_t status;
6000 int retval = 0;
6001
6002 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_resize_t),
6003 mode);
6004 if (retval != 0) {
6005 DERR("srq_resize: copyin error %d\n", retval);
6006 return (EFAULT);
6007 }
6008
6009 /* get srq resource */
6010 srq_rp = (daplka_srq_resource_t *)
6011 daplka_hash_lookup(&ia_rp->ia_srq_htbl, args.srqr_hkey);
6012 if (srq_rp == NULL) {
6013 DERR("srq_resize: cannot find srq resource\n");
6014 return (EINVAL);
6015 }
6016 ASSERT(DAPLKA_RS_TYPE(srq_rp) == DAPL_TYPE_SRQ);
6017
6018 hca_attrp = &ia_rp->ia_hca->hca_attr;
6019 if (args.srqr_new_size > hca_attrp->hca_max_srqs_sz) {
6020 DERR("srq_resize: invalid srq size %d", args.srqr_new_size);
6021 retval = EINVAL;
6022 goto cleanup;
6023 }
6024
6025 mutex_enter(&srq_rp->srq_lock);
6026 /*
6027 * If ibt_resize_srq fails that it is primarily due to resource
6028 * shortage. Per IB spec resize will never loose events and
6029 * a resize error leaves the SRQ intact. Therefore even if the
6030 * resize request fails we proceed and get the mapping data
6031 * from the SRQ so that the library can mmap it.
6032 */
6033 status = ibt_modify_srq(srq_rp->srq_hdl, IBT_SRQ_SET_SIZE,
6034 args.srqr_new_size, 0, &args.srqr_real_size);
6035 if (status != IBT_SUCCESS) {
6036 /* we return the size of the old CQ if resize fails */
6037 args.srqr_real_size = srq_rp->srq_real_size;
6038 ASSERT(status != IBT_SRQ_HDL_INVALID);
6039 DERR("srq_resize: ibt_modify_srq failed:%d\n", status);
6040 } else {
6041 srq_rp->srq_real_size = args.srqr_real_size;
6042 }
6043 mutex_exit(&srq_rp->srq_lock);
6044
6045
6046 D2("srq_resize(%d): done new_sz(%u) real_sz(%u)\n",
6047 DAPLKA_RS_RNUM(srq_rp), args.srqr_new_size, args.srqr_real_size);
6048
6049 /* Get HCA-specific data_out info */
6050 status = ibt_ci_data_out(srq_rp->srq_hca_hdl,
6051 IBT_CI_NO_FLAGS, IBT_HDL_SRQ, (void *)srq_rp->srq_hdl,
6052 &args.srqr_data_out, sizeof (args.srqr_data_out));
6053 if (status != IBT_SUCCESS) {
6054 DERR("srq_resize: ibt_ci_data_out error(%d)\n", status);
6055 /* return ibt_ci_data_out status */
6056 *rvalp = (int)status;
6057 retval = 0;
6058 goto cleanup;
6059 }
6060
6061 retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_srq_resize_t),
6062 mode);
6063 if (retval != 0) {
6064 DERR("srq_resize: copyout error %d\n", retval);
6065 retval = EFAULT;
6066 goto cleanup;
6067 }
6068
6069 cleanup:;
6070 if (srq_rp != NULL) {
6071 DAPLKA_RS_UNREF(srq_rp);
6072 }
6073 return (retval);
6074 }
6075
6076 /*
6077 * Frees an SRQ resource.
6078 */
6079 /* ARGSUSED */
6080 static int
6081 daplka_srq_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
6082 cred_t *cred, int *rvalp)
6083 {
6084 daplka_srq_resource_t *srq_rp = NULL;
6085 dapl_srq_free_t args;
6086 int retval;
6087
6088 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_free_t), mode);
6089 if (retval != 0) {
6090 DERR("srq_free: copyin error %d\n", retval);
6091 return (EFAULT);
6092 }
6093
6094 retval = daplka_hash_remove(&ia_rp->ia_srq_htbl,
6095 args.srqf_hkey, (void **)&srq_rp);
6096 if (retval != 0 || srq_rp == NULL) {
6097 /*
6098 * this is only possible if we have two threads
6099 * calling ep_free in parallel.
6100 */
6101 DERR("srq_free: cannot find resource retval(%d) 0x%llx\n",
6102 retval, args.srqf_hkey);
6103 return (EINVAL);
6104 }
6105
6106 /* UNREF calls the actual free function when refcnt is zero */
6107 DAPLKA_RS_UNREF(srq_rp);
6108 return (0);
6109 }
6110
6111 /*
6112 * destroys a SRQ resource.
6113 * called when refcnt drops to zero.
6114 */
6115 static int
6116 daplka_srq_destroy(daplka_resource_t *gen_rp)
6117 {
6118 daplka_srq_resource_t *srq_rp = (daplka_srq_resource_t *)gen_rp;
6119 ibt_status_t status;
6120
6121 ASSERT(DAPLKA_RS_REFCNT(srq_rp) == 0);
6122
6123 D3("srq_destroy: entering, srq_rp 0x%p, rnum %d\n",
6124 srq_rp, DAPLKA_RS_RNUM(srq_rp));
6125 /*
6126 * destroy the srq
6127 */
6128 if (srq_rp->srq_hdl != NULL) {
6129 status = daplka_ibt_free_srq(srq_rp, srq_rp->srq_hdl);
6130 if (status != IBT_SUCCESS) {
6131 DERR("srq_destroy: ibt_free_srq returned %d\n",
6132 status);
6133 }
6134 srq_rp->srq_hdl = NULL;
6135 D3("srq_destroy: srq freed, rnum %d\n", DAPLKA_RS_RNUM(srq_rp));
6136 }
6137 /*
6138 * release all references
6139 */
6140 if (srq_rp->srq_pd_res != NULL) {
6141 DAPLKA_RS_UNREF(srq_rp->srq_pd_res);
6142 srq_rp->srq_pd_res = NULL;
6143 }
6144
6145 mutex_destroy(&srq_rp->srq_lock);
6146 DAPLKA_RS_FINI(srq_rp);
6147 kmem_free(srq_rp, sizeof (daplka_srq_resource_t));
6148 D3("srq_destroy: exiting, srq_rp 0x%p\n", srq_rp);
6149 return (0);
6150 }
6151
6152 static void
6153 daplka_hash_srq_free(void *obj)
6154 {
6155 daplka_srq_resource_t *srq_rp = (daplka_srq_resource_t *)obj;
6156
6157 ASSERT(DAPLKA_RS_TYPE(srq_rp) == DAPL_TYPE_SRQ);
6158 DAPLKA_RS_UNREF(srq_rp);
6159 }
6160
6161 /*
6162 * This function tells the CM to start listening on a service id.
6163 * It must be called by the passive side client before the client
6164 * can receive connection requests from remote endpoints. If the
6165 * client specifies a non-zero service id (connection qualifier in
6166 * dapl terms), this function will attempt to bind to this service
6167 * id and return an error if the id is already in use. If the client
6168 * specifies zero as the service id, this function will try to find
6169 * the next available service id and return it back to the client.
6170 * To support the cr_handoff function, this function will, in addition
6171 * to creating and inserting an SP resource into the per-IA SP hash
6172 * table, insert the SP resource into a global SP table. This table
6173 * maintains all active service points created by all dapl clients.
6174 * CR handoff locates the target SP by iterating through this global
6175 * table.
6176 */
6177 /* ARGSUSED */
6178 static int
6179 daplka_service_register(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
6180 cred_t *cred, int *rvalp)
6181 {
6182 daplka_evd_resource_t *evd_rp = NULL;
6183 daplka_sp_resource_t *sp_rp = NULL;
6184 dapl_service_register_t args;
6185 ibt_srv_desc_t sd_args;
6186 ibt_srv_bind_t sb_args;
6187 ibt_status_t status;
6188 ib_svc_id_t retsid = 0;
6189 uint64_t sp_hkey = 0;
6190 boolean_t bumped = B_FALSE;
6191 int backlog_size;
6192 int retval = 0;
6193
6194 retval = ddi_copyin((void *)arg, &args,
6195 sizeof (dapl_service_register_t), mode);
6196 if (retval != 0) {
6197 DERR("service_register: copyin error %d\n", retval);
6198 return (EINVAL);
6199 }
6200
6201 sp_rp = kmem_zalloc(sizeof (*sp_rp), daplka_km_flags);
6202 if (sp_rp == NULL) {
6203 DERR("service_register: cannot allocate sp resource\n");
6204 return (ENOMEM);
6205 }
6206 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sp_rp))
6207 DAPLKA_RS_INIT(sp_rp, DAPL_TYPE_SP,
6208 DAPLKA_RS_RNUM(ia_rp), daplka_sp_destroy);
6209
6210 /* check if evd exists */
6211 evd_rp = (daplka_evd_resource_t *)
6212 daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.sr_evd_hkey);
6213 if (evd_rp == NULL) {
6214 DERR("service_register: evd resource not found\n");
6215 retval = EINVAL;
6216 goto cleanup;
6217 }
6218 /*
6219 * initialize backlog size
6220 */
6221 if (evd_rp && evd_rp->evd_cq_real_size > 0) {
6222 backlog_size = evd_rp->evd_cq_real_size + 1;
6223 } else {
6224 backlog_size = DAPLKA_DEFAULT_SP_BACKLOG;
6225 }
6226 D2("service_register: args.sr_sid = %llu\n", (longlong_t)args.sr_sid);
6227
6228 /* save the userland sp ptr */
6229 sp_rp->sp_cookie = args.sr_sp_cookie;
6230 sp_rp->sp_backlog_size = backlog_size;
6231 D3("service_register: backlog set to %d\n", sp_rp->sp_backlog_size);
6232 sp_rp->sp_backlog = kmem_zalloc(sp_rp->sp_backlog_size *
6233 sizeof (daplka_sp_conn_pend_t), daplka_km_flags);
6234
6235 /* save evd resource pointer */
6236 sp_rp->sp_evd_res = evd_rp;
6237
6238 /*
6239 * save ruid here so that we can do a comparison later
6240 * when someone does cr_handoff. the check will prevent
6241 * a malicious app from passing a CR to us.
6242 */
6243 sp_rp->sp_ruid = crgetruid(cred);
6244
6245 /* fill in args for register_service */
6246 sd_args.sd_ud_handler = NULL;
6247 sd_args.sd_handler = daplka_cm_service_handler;
6248 sd_args.sd_flags = IBT_SRV_NO_FLAGS;
6249
6250 status = ibt_register_service(daplka_dev->daplka_clnt_hdl,
6251 &sd_args, args.sr_sid, 1, &sp_rp->sp_srv_hdl, &retsid);
6252
6253 if (status != IBT_SUCCESS) {
6254 DERR("service_register: ibt_register_service returned %d\n",
6255 status);
6256 *rvalp = (int)status;
6257 retval = 0;
6258 goto cleanup;
6259 }
6260 /* save returned sid */
6261 sp_rp->sp_conn_qual = retsid;
6262 args.sr_retsid = retsid;
6263
6264 /* fill in args for bind_service */
6265 sb_args.sb_pkey = ia_rp->ia_port_pkey;
6266 sb_args.sb_lease = 0xffffffff;
6267 sb_args.sb_key[0] = 0x1234;
6268 sb_args.sb_key[1] = 0x5678;
6269 sb_args.sb_name = DAPLKA_DRV_NAME;
6270
6271 D2("service_register: bind(0x%llx:0x%llx)\n",
6272 (longlong_t)ia_rp->ia_hca_sgid.gid_prefix,
6273 (longlong_t)ia_rp->ia_hca_sgid.gid_guid);
6274
6275 status = ibt_bind_service(sp_rp->sp_srv_hdl, ia_rp->ia_hca_sgid,
6276 &sb_args, (void *)sp_rp, &sp_rp->sp_bind_hdl);
6277 if (status != IBT_SUCCESS) {
6278 DERR("service_register: ibt_bind_service returned %d\n",
6279 status);
6280 *rvalp = (int)status;
6281 retval = 0;
6282 goto cleanup;
6283 }
6284
6285 /*
6286 * need to bump refcnt because the global hash table will
6287 * have a reference to sp_rp
6288 */
6289 DAPLKA_RS_REF(sp_rp);
6290 bumped = B_TRUE;
6291
6292 /* insert into global sp hash table */
6293 sp_rp->sp_global_hkey = 0;
6294 retval = daplka_hash_insert(&daplka_global_sp_htbl,
6295 &sp_rp->sp_global_hkey, (void *)sp_rp);
6296 if (retval != 0) {
6297 DERR("service_register: cannot insert sp resource\n");
6298 goto cleanup;
6299 }
6300 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sp_rp))
6301
6302 /* insert into per-IA sp hash table */
6303 retval = daplka_hash_insert(&ia_rp->ia_sp_htbl,
6304 &sp_hkey, (void *)sp_rp);
6305 if (retval != 0) {
6306 DERR("service_register: cannot insert sp resource\n");
6307 goto cleanup;
6308 }
6309
6310 /* pass index to application */
6311 args.sr_sp_hkey = sp_hkey;
6312 retval = ddi_copyout(&args, (void *)arg,
6313 sizeof (dapl_service_register_t), mode);
6314 if (retval != 0) {
6315 DERR("service_register: copyout error %d\n", retval);
6316 retval = EFAULT;
6317 goto cleanup;
6318 }
6319 return (0);
6320
6321 cleanup:;
6322 ASSERT(sp_rp != NULL);
6323 /* remove from ia table */
6324 if (sp_hkey != 0) {
6325 daplka_sp_resource_t *free_rp = NULL;
6326
6327 (void) daplka_hash_remove(&ia_rp->ia_sp_htbl,
6328 sp_hkey, (void **)&free_rp);
6329 if (free_rp != sp_rp) {
6330 DERR("service_register: cannot remove sp\n");
6331 /*
6332 * we can only get here if another thread
6333 * has completed the cleanup in svc_deregister
6334 */
6335 return (retval);
6336 }
6337 }
6338
6339 /* remove from global table */
6340 if (sp_rp->sp_global_hkey != 0) {
6341 daplka_sp_resource_t *free_rp = NULL;
6342
6343 /*
6344 * we get here if either the hash_insert into
6345 * ia_sp_htbl failed or the ddi_copyout failed.
6346 * hash_insert failure implies that we are the
6347 * only thread with a reference to sp. ddi_copyout
6348 * failure implies that svc_deregister could have
6349 * picked up the sp and destroyed it. but since
6350 * we got to this point, we must have removed
6351 * the sp ourselves in hash_remove above and
6352 * that the sp can be destroyed by us.
6353 */
6354 (void) daplka_hash_remove(&daplka_global_sp_htbl,
6355 sp_rp->sp_global_hkey, (void **)&free_rp);
6356 if (free_rp != sp_rp) {
6357 DERR("service_register: cannot remove sp\n");
6358 /*
6359 * this case is impossible. see explanation above.
6360 */
6361 ASSERT(B_FALSE);
6362 return (retval);
6363 }
6364 sp_rp->sp_global_hkey = 0;
6365 }
6366 /* unreference sp */
6367 if (bumped) {
6368 DAPLKA_RS_UNREF(sp_rp);
6369 }
6370
6371 /* destroy sp resource */
6372 DAPLKA_RS_UNREF(sp_rp);
6373 return (retval);
6374 }
6375
6376 /*
6377 * deregisters the service and removes SP from the global table.
6378 */
6379 /* ARGSUSED */
6380 static int
6381 daplka_service_deregister(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
6382 cred_t *cred, int *rvalp)
6383 {
6384 dapl_service_deregister_t args;
6385 daplka_sp_resource_t *sp_rp = NULL, *g_sp_rp = NULL;
6386 int retval;
6387
6388 retval = ddi_copyin((void *)arg, &args,
6389 sizeof (dapl_service_deregister_t), mode);
6390
6391 if (retval != 0) {
6392 DERR("service_deregister: copyin error %d\n", retval);
6393 return (EINVAL);
6394 }
6395
6396 retval = daplka_hash_remove(&ia_rp->ia_sp_htbl,
6397 args.sdr_sp_hkey, (void **)&sp_rp);
6398 if (retval != 0 || sp_rp == NULL) {
6399 DERR("service_deregister: cannot find sp resource\n");
6400 return (EINVAL);
6401 }
6402
6403 retval = daplka_hash_remove(&daplka_global_sp_htbl,
6404 sp_rp->sp_global_hkey, (void **)&g_sp_rp);
6405 if (retval != 0 || g_sp_rp == NULL) {
6406 DERR("service_deregister: cannot find sp resource\n");
6407 }
6408
6409 /* remove the global reference */
6410 if (g_sp_rp == sp_rp) {
6411 DAPLKA_RS_UNREF(g_sp_rp);
6412 }
6413
6414 DAPLKA_RS_UNREF(sp_rp);
6415 return (0);
6416 }
6417
6418 /*
6419 * destroys a service point.
6420 * called when the refcnt drops to zero.
6421 */
6422 static int
6423 daplka_sp_destroy(daplka_resource_t *gen_rp)
6424 {
6425 daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)gen_rp;
6426 ibt_status_t status;
6427
6428 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sp_rp))
6429 ASSERT(DAPLKA_RS_REFCNT(sp_rp) == 0);
6430 D3("sp_destroy: entering, sp_rp %p, rnum %d\n",
6431 sp_rp, DAPLKA_RS_RNUM(sp_rp));
6432
6433 /*
6434 * it is possible for pending connections to remain
6435 * on an SP. We need to clean them up here.
6436 */
6437 if (sp_rp->sp_backlog != NULL) {
6438 ibt_cm_proceed_reply_t proc_reply;
6439 int i, cnt = 0;
6440 void *spcp_sidp;
6441
6442 for (i = 0; i < sp_rp->sp_backlog_size; i++) {
6443 if (sp_rp->sp_backlog[i].spcp_state ==
6444 DAPLKA_SPCP_PENDING) {
6445 cnt++;
6446 if (sp_rp->sp_backlog[i].spcp_sid == NULL) {
6447 DERR("sp_destroy: "
6448 "spcp_sid == NULL!\n");
6449 continue;
6450 }
6451 mutex_enter(&sp_rp->sp_lock);
6452 spcp_sidp = sp_rp->sp_backlog[i].spcp_sid;
6453 sp_rp->sp_backlog[i].spcp_state =
6454 DAPLKA_SPCP_INIT;
6455 sp_rp->sp_backlog[i].spcp_sid = NULL;
6456 sp_rp->sp_backlog[i].spcp_req_len = 0;
6457 mutex_exit(&sp_rp->sp_lock);
6458 status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV,
6459 spcp_sidp,
6460 IBT_CM_NO_RESOURCE, &proc_reply, NULL, 0);
6461 if (status != IBT_SUCCESS) {
6462 DERR("sp_destroy: proceed failed %d\n",
6463 status);
6464 }
6465 }
6466 }
6467 if (cnt > 0) {
6468 DERR("sp_destroy: found %d pending "
6469 "connections\n", cnt);
6470 }
6471 }
6472
6473 if (sp_rp->sp_srv_hdl != NULL && sp_rp->sp_bind_hdl != NULL) {
6474 status = ibt_unbind_service(sp_rp->sp_srv_hdl,
6475 sp_rp->sp_bind_hdl);
6476 if (status != IBT_SUCCESS) {
6477 DERR("sp_destroy: ibt_unbind_service "
6478 "failed: %d\n", status);
6479 }
6480 }
6481
6482 if (sp_rp->sp_srv_hdl != NULL) {
6483 status = ibt_deregister_service(daplka_dev->daplka_clnt_hdl,
6484 sp_rp->sp_srv_hdl);
6485 if (status != IBT_SUCCESS) {
6486 DERR("sp_destroy: ibt_deregister_service "
6487 "failed: %d\n", status);
6488 }
6489 }
6490 if (sp_rp->sp_backlog != NULL) {
6491 kmem_free(sp_rp->sp_backlog,
6492 sp_rp->sp_backlog_size * sizeof (daplka_sp_conn_pend_t));
6493 sp_rp->sp_backlog = NULL;
6494 sp_rp->sp_backlog_size = 0;
6495 }
6496
6497 /*
6498 * release reference to evd
6499 */
6500 if (sp_rp->sp_evd_res != NULL) {
6501 DAPLKA_RS_UNREF(sp_rp->sp_evd_res);
6502 }
6503 sp_rp->sp_bind_hdl = NULL;
6504 sp_rp->sp_srv_hdl = NULL;
6505 DAPLKA_RS_FINI(sp_rp);
6506 kmem_free(sp_rp, sizeof (*sp_rp));
6507 D3("sp_destroy: exiting, sp_rp %p\n", sp_rp);
6508 return (0);
6509 }
6510
6511 /*
6512 * this function is called by daplka_hash_destroy for
6513 * freeing SP resource objects
6514 */
6515 static void
6516 daplka_hash_sp_free(void *obj)
6517 {
6518 daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)obj;
6519 daplka_sp_resource_t *g_sp_rp;
6520 int retval;
6521
6522 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
6523
6524 retval = daplka_hash_remove(&daplka_global_sp_htbl,
6525 sp_rp->sp_global_hkey, (void **)&g_sp_rp);
6526 if (retval != 0 || g_sp_rp == NULL) {
6527 DERR("sp_free: cannot find sp resource\n");
6528 }
6529 if (g_sp_rp == sp_rp) {
6530 DAPLKA_RS_UNREF(g_sp_rp);
6531 }
6532
6533 DAPLKA_RS_UNREF(sp_rp);
6534 }
6535
6536 static void
6537 daplka_hash_sp_unref(void *obj)
6538 {
6539 daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)obj;
6540
6541 ASSERT(DAPLKA_RS_TYPE(sp_rp) == DAPL_TYPE_SP);
6542 DAPLKA_RS_UNREF(sp_rp);
6543 }
6544
6545 /*
6546 * Passive side CM handlers
6547 */
6548
6549 /*
6550 * processes the REQ_RCV event
6551 */
6552 /* ARGSUSED */
6553 static ibt_cm_status_t
6554 daplka_cm_service_req(daplka_sp_resource_t *spp, ibt_cm_event_t *event,
6555 ibt_cm_return_args_t *ret_args, void *pr_data, ibt_priv_data_len_t pr_len)
6556 {
6557 daplka_sp_conn_pend_t *conn = NULL;
6558 daplka_evd_event_t *cr_ev = NULL;
6559 ibt_cm_status_t cm_status = IBT_CM_DEFAULT;
6560 uint16_t bkl_index;
6561 ibt_status_t status;
6562
6563 /*
6564 * acquire a slot in the connection backlog of this service point
6565 */
6566 mutex_enter(&spp->sp_lock);
6567 for (bkl_index = 0; bkl_index < spp->sp_backlog_size; bkl_index++) {
6568 if (spp->sp_backlog[bkl_index].spcp_state == DAPLKA_SPCP_INIT) {
6569 conn = &spp->sp_backlog[bkl_index];
6570 ASSERT(conn->spcp_sid == NULL);
6571 conn->spcp_state = DAPLKA_SPCP_PENDING;
6572 conn->spcp_sid = event->cm_session_id;
6573 break;
6574 }
6575 }
6576 mutex_exit(&spp->sp_lock);
6577
6578 /*
6579 * too many pending connections
6580 */
6581 if (bkl_index == spp->sp_backlog_size) {
6582 DERR("service_req: connection pending exceeded %d limit\n",
6583 spp->sp_backlog_size);
6584 return (IBT_CM_NO_RESOURCE);
6585 }
6586 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*conn))
6587
6588 /*
6589 * save data for cr_handoff
6590 */
6591 if (pr_data != NULL && pr_len > 0) {
6592 int trunc_len = pr_len;
6593
6594 if (trunc_len > DAPL_MAX_PRIVATE_DATA_SIZE) {
6595 DERR("service_req: private data truncated\n");
6596 trunc_len = DAPL_MAX_PRIVATE_DATA_SIZE;
6597 }
6598 conn->spcp_req_len = trunc_len;
6599 bcopy(pr_data, conn->spcp_req_data, trunc_len);
6600 } else {
6601 conn->spcp_req_len = 0;
6602 }
6603 conn->spcp_rdma_ra_in = event->cm_event.req.req_rdma_ra_in;
6604 conn->spcp_rdma_ra_out = event->cm_event.req.req_rdma_ra_out;
6605
6606 /*
6607 * create a CR event
6608 */
6609 cr_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
6610 if (cr_ev == NULL) {
6611 DERR("service_req: could not alloc cr_ev\n");
6612 cm_status = IBT_CM_NO_RESOURCE;
6613 goto cleanup;
6614 }
6615
6616 cr_ev->ee_next = NULL;
6617 cr_ev->ee_cmev.ec_cm_cookie = spp->sp_cookie;
6618 cr_ev->ee_cmev.ec_cm_is_passive = B_TRUE;
6619 cr_ev->ee_cmev.ec_cm_psep_cookie = DAPLKA_CREATE_PSEP_COOKIE(bkl_index);
6620 /*
6621 * save the requestor gid
6622 * daplka_event_poll needs this if this is a third party REQ_RCV
6623 */
6624 cr_ev->ee_cmev.ec_cm_req_prim_addr.gid_prefix =
6625 event->cm_event.req.req_prim_addr.av_dgid.gid_prefix;
6626 cr_ev->ee_cmev.ec_cm_req_prim_addr.gid_guid =
6627 event->cm_event.req.req_prim_addr.av_dgid.gid_guid;
6628
6629 /*
6630 * set event type
6631 */
6632 if (pr_len == 0) {
6633 cr_ev->ee_cmev.ec_cm_ev_type =
6634 DAPL_IB_CME_CONNECTION_REQUEST_PENDING;
6635 } else {
6636 cr_ev->ee_cmev.ec_cm_ev_priv_data =
6637 kmem_zalloc(pr_len, KM_NOSLEEP);
6638 if (cr_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
6639 DERR("service_req: could not alloc priv\n");
6640 cm_status = IBT_CM_NO_RESOURCE;
6641 goto cleanup;
6642 }
6643 bcopy(pr_data, cr_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
6644 cr_ev->ee_cmev.ec_cm_ev_type =
6645 DAPL_IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA;
6646 }
6647 cr_ev->ee_cmev.ec_cm_ev_priv_data_len = pr_len;
6648
6649 /*
6650 * tell the active side to expect the processing time to be
6651 * at most equal to daplka_cm_delay
6652 */
6653 status = ibt_cm_delay(IBT_CM_DELAY_REQ, event->cm_session_id,
6654 daplka_cm_delay, NULL, 0);
6655 if (status != IBT_SUCCESS) {
6656 DERR("service_req: ibt_cm_delay failed %d\n", status);
6657 cm_status = IBT_CM_NO_RESOURCE;
6658 goto cleanup;
6659 }
6660
6661 /*
6662 * enqueue cr_ev onto the cr_events list of the EVD
6663 * corresponding to the SP
6664 */
6665 D2("service_req: enqueue event(%p) evdp(%p) priv_data(%p) "
6666 "priv_len(%d) psep(0x%llx)\n", cr_ev, spp->sp_evd_res,
6667 cr_ev->ee_cmev.ec_cm_ev_priv_data,
6668 (int)cr_ev->ee_cmev.ec_cm_ev_priv_data_len,
6669 (longlong_t)cr_ev->ee_cmev.ec_cm_psep_cookie);
6670
6671 daplka_evd_wakeup(spp->sp_evd_res,
6672 &spp->sp_evd_res->evd_cr_events, cr_ev);
6673
6674 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*conn))
6675 return (IBT_CM_DEFER);
6676
6677 cleanup:;
6678 /*
6679 * free the cr event
6680 */
6681 if (cr_ev != NULL) {
6682 if (cr_ev->ee_cmev.ec_cm_ev_priv_data != NULL) {
6683 kmem_free(cr_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
6684 cr_ev->ee_cmev.ec_cm_ev_priv_data = NULL;
6685 cr_ev->ee_cmev.ec_cm_ev_priv_data_len = 0;
6686 }
6687 kmem_free(cr_ev, sizeof (daplka_evd_event_t));
6688 }
6689 /*
6690 * release our slot in the backlog array
6691 */
6692 if (conn != NULL) {
6693 mutex_enter(&spp->sp_lock);
6694 ASSERT(conn->spcp_state == DAPLKA_SPCP_PENDING);
6695 ASSERT(conn->spcp_sid == event->cm_session_id);
6696 conn->spcp_state = DAPLKA_SPCP_INIT;
6697 conn->spcp_req_len = 0;
6698 conn->spcp_sid = NULL;
6699 mutex_exit(&spp->sp_lock);
6700 }
6701 return (cm_status);
6702 }
6703
6704 /*
6705 * processes the CONN_CLOSED event
6706 */
6707 /* ARGSUSED */
6708 static ibt_cm_status_t
6709 daplka_cm_service_conn_closed(daplka_sp_resource_t *sp_rp,
6710 ibt_cm_event_t *event, ibt_cm_return_args_t *ret_args,
6711 void *priv_data, ibt_priv_data_len_t len)
6712 {
6713 daplka_ep_resource_t *ep_rp;
6714 daplka_evd_event_t *disc_ev;
6715 uint32_t old_state, new_state;
6716
6717 ep_rp = (daplka_ep_resource_t *)
6718 ibt_get_chan_private(event->cm_channel);
6719 if (ep_rp == NULL) {
6720 DERR("service_conn_closed: ep_rp == NULL\n");
6721 return (IBT_CM_ACCEPT);
6722 }
6723
6724 /*
6725 * verify that the ep_state is either CONNECTED or
6726 * DISCONNECTING. if it is not in either states return
6727 * without generating an event.
6728 */
6729 new_state = old_state = daplka_ep_get_state(ep_rp);
6730 if (old_state != DAPLKA_EP_STATE_CONNECTED &&
6731 old_state != DAPLKA_EP_STATE_DISCONNECTING) {
6732 /*
6733 * we can get here if the connection is being aborted
6734 */
6735 D2("service_conn_closed: conn aborted, state = %d, "
6736 "closed = %d\n", old_state, (int)event->cm_event.closed);
6737 daplka_ep_set_state(ep_rp, old_state, new_state);
6738 return (IBT_CM_ACCEPT);
6739 }
6740
6741 /*
6742 * create a DAPL_IB_CME_DISCONNECTED event
6743 */
6744 disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
6745 if (disc_ev == NULL) {
6746 DERR("service_conn_closed: cannot alloc disc_ev\n");
6747 daplka_ep_set_state(ep_rp, old_state, new_state);
6748 return (IBT_CM_ACCEPT);
6749 }
6750
6751 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_DISCONNECTED;
6752 disc_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
6753 disc_ev->ee_cmev.ec_cm_is_passive = B_TRUE;
6754 disc_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
6755 disc_ev->ee_cmev.ec_cm_ev_priv_data = NULL;
6756 disc_ev->ee_cmev.ec_cm_ev_priv_data_len = 0;
6757
6758 D2("service_conn_closed: enqueue event(%p) evdp(%p) psep(0x%llx)\n",
6759 disc_ev, sp_rp->sp_evd_res, (longlong_t)ep_rp->ep_psep_cookie);
6760
6761 /*
6762 * transition ep_state to DISCONNECTED
6763 */
6764 new_state = DAPLKA_EP_STATE_DISCONNECTED;
6765 daplka_ep_set_state(ep_rp, old_state, new_state);
6766
6767 /*
6768 * enqueue event onto the conn_evd owned by ep_rp
6769 */
6770 daplka_evd_wakeup(ep_rp->ep_conn_evd,
6771 &ep_rp->ep_conn_evd->evd_conn_events, disc_ev);
6772
6773 return (IBT_CM_ACCEPT);
6774 }
6775
6776 /*
6777 * processes the CONN_EST event
6778 */
6779 /* ARGSUSED */
6780 static ibt_cm_status_t
6781 daplka_cm_service_conn_est(daplka_sp_resource_t *sp_rp, ibt_cm_event_t *event,
6782 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
6783 {
6784 daplka_ep_resource_t *ep_rp;
6785 daplka_evd_event_t *conn_ev;
6786 void *pr_data = event->cm_priv_data;
6787 ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
6788 uint32_t old_state, new_state;
6789
6790 ep_rp = (daplka_ep_resource_t *)
6791 ibt_get_chan_private(event->cm_channel);
6792 if (ep_rp == NULL) {
6793 DERR("service_conn_est: ep_rp == NULL\n");
6794 return (IBT_CM_ACCEPT);
6795 }
6796
6797 /*
6798 * verify that ep_state is ACCEPTING. if it is not in this
6799 * state, return without generating an event.
6800 */
6801 new_state = old_state = daplka_ep_get_state(ep_rp);
6802 if (old_state != DAPLKA_EP_STATE_ACCEPTING) {
6803 /*
6804 * we can get here if the connection is being aborted
6805 */
6806 DERR("service_conn_est: conn aborted, state = %d\n",
6807 old_state);
6808 daplka_ep_set_state(ep_rp, old_state, new_state);
6809 return (IBT_CM_ACCEPT);
6810 }
6811
6812 /*
6813 * create a DAPL_IB_CME_CONNECTED event
6814 */
6815 conn_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
6816 if (conn_ev == NULL) {
6817 DERR("service_conn_est: conn_ev alloc failed\n");
6818 daplka_ep_set_state(ep_rp, old_state, new_state);
6819 return (IBT_CM_ACCEPT);
6820 }
6821
6822 conn_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_CONNECTED;
6823 conn_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
6824 conn_ev->ee_cmev.ec_cm_is_passive = B_TRUE;
6825 conn_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
6826
6827 /*
6828 * copy private data into event
6829 */
6830 if (pr_len > 0) {
6831 conn_ev->ee_cmev.ec_cm_ev_priv_data =
6832 kmem_zalloc(pr_len, KM_NOSLEEP);
6833 if (conn_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
6834 DERR("service_conn_est: pr_data alloc failed\n");
6835 daplka_ep_set_state(ep_rp, old_state, new_state);
6836 kmem_free(conn_ev, sizeof (daplka_evd_event_t));
6837 return (IBT_CM_ACCEPT);
6838 }
6839 bcopy(pr_data, conn_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
6840 }
6841 conn_ev->ee_cmev.ec_cm_ev_priv_data_len = pr_len;
6842
6843 D2("service_conn_est: enqueue event(%p) evdp(%p)\n",
6844 conn_ev, ep_rp->ep_conn_evd);
6845
6846 /*
6847 * transition ep_state to CONNECTED
6848 */
6849 new_state = DAPLKA_EP_STATE_CONNECTED;
6850 daplka_ep_set_state(ep_rp, old_state, new_state);
6851
6852 /*
6853 * enqueue event onto the conn_evd owned by ep_rp
6854 */
6855 daplka_evd_wakeup(ep_rp->ep_conn_evd,
6856 &ep_rp->ep_conn_evd->evd_conn_events, conn_ev);
6857
6858 return (IBT_CM_ACCEPT);
6859 }
6860
6861 /*
6862 * processes the FAILURE event
6863 */
6864 /* ARGSUSED */
6865 static ibt_cm_status_t
6866 daplka_cm_service_event_failure(daplka_sp_resource_t *sp_rp,
6867 ibt_cm_event_t *event, ibt_cm_return_args_t *ret_args, void *priv_data,
6868 ibt_priv_data_len_t len)
6869 {
6870 daplka_evd_event_t *disc_ev;
6871 daplka_ep_resource_t *ep_rp;
6872 uint32_t old_state, new_state;
6873 ibt_rc_chan_query_attr_t chan_attrs;
6874 ibt_status_t status;
6875
6876 /*
6877 * check that we still have a valid cm_channel before continuing
6878 */
6879 if (event->cm_channel == NULL) {
6880 DERR("serice_event_failure: event->cm_channel == NULL\n");
6881 return (IBT_CM_ACCEPT);
6882 }
6883 ep_rp = (daplka_ep_resource_t *)
6884 ibt_get_chan_private(event->cm_channel);
6885 if (ep_rp == NULL) {
6886 DERR("service_event_failure: ep_rp == NULL\n");
6887 return (IBT_CM_ACCEPT);
6888 }
6889
6890 /*
6891 * verify that ep_state is ACCEPTING or DISCONNECTING. if it
6892 * is not in either state, return without generating an event.
6893 */
6894 new_state = old_state = daplka_ep_get_state(ep_rp);
6895 if (old_state != DAPLKA_EP_STATE_ACCEPTING &&
6896 old_state != DAPLKA_EP_STATE_DISCONNECTING) {
6897 /*
6898 * we can get here if the connection is being aborted
6899 */
6900 DERR("service_event_failure: conn aborted, state = %d, "
6901 "cf_code = %d, cf_msg = %d, cf_reason = %d\n", old_state,
6902 (int)event->cm_event.failed.cf_code,
6903 (int)event->cm_event.failed.cf_msg,
6904 (int)event->cm_event.failed.cf_reason);
6905
6906 daplka_ep_set_state(ep_rp, old_state, new_state);
6907 return (IBT_CM_ACCEPT);
6908 }
6909
6910 bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
6911 status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
6912
6913 if ((status == IBT_SUCCESS) &&
6914 (chan_attrs.rc_state != IBT_STATE_ERROR)) {
6915 DERR("service_event_failure: conn abort qpn %d state %d\n",
6916 chan_attrs.rc_qpn, chan_attrs.rc_state);
6917
6918 /* explicit transition the QP to ERROR state */
6919 status = ibt_flush_channel(ep_rp->ep_chan_hdl);
6920 }
6921
6922 /*
6923 * create an event
6924 */
6925 disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
6926 if (disc_ev == NULL) {
6927 DERR("service_event_failure: cannot alloc disc_ev\n");
6928 daplka_ep_set_state(ep_rp, old_state, new_state);
6929 return (IBT_CM_ACCEPT);
6930 }
6931
6932 /*
6933 * fill in the appropriate event type
6934 */
6935 if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_TIMEOUT) {
6936 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
6937 } else if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_REJ_RCV) {
6938 switch (event->cm_event.failed.cf_reason) {
6939 case IBT_CM_INVALID_CID:
6940 disc_ev->ee_cmev.ec_cm_ev_type =
6941 DAPL_IB_CME_DESTINATION_REJECT;
6942 break;
6943 default:
6944 disc_ev->ee_cmev.ec_cm_ev_type =
6945 DAPL_IB_CME_LOCAL_FAILURE;
6946 break;
6947 }
6948 } else {
6949 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_LOCAL_FAILURE;
6950 }
6951 disc_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
6952 disc_ev->ee_cmev.ec_cm_is_passive = B_TRUE;
6953 disc_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
6954 disc_ev->ee_cmev.ec_cm_ev_priv_data_len = 0;
6955 disc_ev->ee_cmev.ec_cm_ev_priv_data = NULL;
6956
6957 D2("service_event_failure: enqueue event(%p) evdp(%p) cf_code(%d) "
6958 "cf_msg(%d) cf_reason(%d) psep(0x%llx)\n", disc_ev,
6959 ep_rp->ep_conn_evd, (int)event->cm_event.failed.cf_code,
6960 (int)event->cm_event.failed.cf_msg,
6961 (int)event->cm_event.failed.cf_reason,
6962 (longlong_t)ep_rp->ep_psep_cookie);
6963
6964 /*
6965 * transition ep_state to DISCONNECTED
6966 */
6967 new_state = DAPLKA_EP_STATE_DISCONNECTED;
6968 daplka_ep_set_state(ep_rp, old_state, new_state);
6969
6970 /*
6971 * enqueue event onto the conn_evd owned by ep_rp
6972 */
6973 daplka_evd_wakeup(ep_rp->ep_conn_evd,
6974 &ep_rp->ep_conn_evd->evd_conn_events, disc_ev);
6975
6976 return (IBT_CM_ACCEPT);
6977 }
6978
6979 /*
6980 * this is the passive side CM handler. it gets registered
6981 * when an SP resource is created in daplka_service_register.
6982 */
6983 static ibt_cm_status_t
6984 daplka_cm_service_handler(void *cm_private, ibt_cm_event_t *event,
6985 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
6986 {
6987 daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)cm_private;
6988
6989 if (sp_rp == NULL) {
6990 DERR("service_handler: sp_rp == NULL\n");
6991 return (IBT_CM_NO_RESOURCE);
6992 }
6993 /*
6994 * default is not to return priv data
6995 */
6996 if (ret_args != NULL) {
6997 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ret_args))
6998 ret_args->cm_ret_len = 0;
6999 }
7000
7001 switch (event->cm_type) {
7002 case IBT_CM_EVENT_REQ_RCV:
7003 D2("service_handler: IBT_CM_EVENT_REQ_RCV\n");
7004 return (daplka_cm_service_req(sp_rp, event, ret_args,
7005 event->cm_priv_data, event->cm_priv_data_len));
7006
7007 case IBT_CM_EVENT_REP_RCV:
7008 /* passive side should not receive this event */
7009 D2("service_handler: IBT_CM_EVENT_REP_RCV\n");
7010 return (IBT_CM_DEFAULT);
7011
7012 case IBT_CM_EVENT_CONN_CLOSED:
7013 D2("service_handler: IBT_CM_EVENT_CONN_CLOSED %d\n",
7014 event->cm_event.closed);
7015 return (daplka_cm_service_conn_closed(sp_rp, event, ret_args,
7016 priv_data, len));
7017
7018 case IBT_CM_EVENT_MRA_RCV:
7019 /* passive side does default processing MRA event */
7020 D2("service_handler: IBT_CM_EVENT_MRA_RCV\n");
7021 return (IBT_CM_DEFAULT);
7022
7023 case IBT_CM_EVENT_CONN_EST:
7024 D2("service_handler: IBT_CM_EVENT_CONN_EST\n");
7025 return (daplka_cm_service_conn_est(sp_rp, event, ret_args,
7026 priv_data, len));
7027
7028 case IBT_CM_EVENT_FAILURE:
7029 D2("service_handler: IBT_CM_EVENT_FAILURE\n");
7030 return (daplka_cm_service_event_failure(sp_rp, event, ret_args,
7031 priv_data, len));
7032 case IBT_CM_EVENT_LAP_RCV:
7033 /* active side had initiated a path migration operation */
7034 D2("service_handler: IBT_CM_EVENT_LAP_RCV\n");
7035 return (IBT_CM_ACCEPT);
7036 default:
7037 DERR("service_handler: invalid event %d\n", event->cm_type);
7038 break;
7039 }
7040 return (IBT_CM_DEFAULT);
7041 }
7042
7043 /*
7044 * Active side CM handlers
7045 */
7046
7047 /*
7048 * Processes the REP_RCV event. When the passive side accepts the
7049 * connection, this handler is called. We make a copy of the private
7050 * data into the ep so that it can be passed back to userland in when
7051 * the CONN_EST event occurs.
7052 */
7053 /* ARGSUSED */
7054 static ibt_cm_status_t
7055 daplka_cm_rc_rep_rcv(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
7056 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
7057 {
7058 void *pr_data = event->cm_priv_data;
7059 ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
7060 uint32_t old_state, new_state;
7061
7062 D2("rc_rep_rcv: pr_data(0x%p), pr_len(%d)\n", pr_data,
7063 (int)pr_len);
7064
7065 ASSERT(ep_rp != NULL);
7066 new_state = old_state = daplka_ep_get_state(ep_rp);
7067 if (old_state != DAPLKA_EP_STATE_CONNECTING) {
7068 /*
7069 * we can get here if the connection is being aborted
7070 */
7071 DERR("rc_rep_rcv: conn aborted, state = %d\n", old_state);
7072 daplka_ep_set_state(ep_rp, old_state, new_state);
7073 return (IBT_CM_NO_CHANNEL);
7074 }
7075
7076 /*
7077 * we do not cancel the timer here because the connection
7078 * handshake is still in progress.
7079 */
7080
7081 /*
7082 * save the private data. it will be passed up when
7083 * the connection is established.
7084 */
7085 if (pr_len > 0) {
7086 ep_rp->ep_priv_len = pr_len;
7087 bcopy(pr_data, ep_rp->ep_priv_data, (size_t)pr_len);
7088 }
7089
7090 /*
7091 * we do not actually transition to a different state.
7092 * the state will change when we get a conn_est, failure,
7093 * closed, or timeout event.
7094 */
7095 daplka_ep_set_state(ep_rp, old_state, new_state);
7096 return (IBT_CM_ACCEPT);
7097 }
7098
7099 /*
7100 * Processes the CONN_CLOSED event. This gets called when either
7101 * the active or passive side closes the rc channel.
7102 */
7103 /* ARGSUSED */
7104 static ibt_cm_status_t
7105 daplka_cm_rc_conn_closed(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
7106 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
7107 {
7108 daplka_evd_event_t *disc_ev;
7109 uint32_t old_state, new_state;
7110
7111 ASSERT(ep_rp != NULL);
7112 old_state = new_state = daplka_ep_get_state(ep_rp);
7113 if (old_state != DAPLKA_EP_STATE_CONNECTED &&
7114 old_state != DAPLKA_EP_STATE_DISCONNECTING) {
7115 /*
7116 * we can get here if the connection is being aborted
7117 */
7118 D2("rc_conn_closed: conn aborted, state = %d, "
7119 "closed = %d\n", old_state, (int)event->cm_event.closed);
7120 daplka_ep_set_state(ep_rp, old_state, new_state);
7121 return (IBT_CM_ACCEPT);
7122 }
7123
7124 /*
7125 * it's ok for the timer to fire at this point. the
7126 * taskq thread that processes the timer will just wait
7127 * until we are done with our state transition.
7128 */
7129 if (daplka_cancel_timer(ep_rp) != 0) {
7130 /*
7131 * daplka_cancel_timer returns -1 if the timer is
7132 * being processed and 0 for all other cases.
7133 * we need to reset ep_state to allow timer processing
7134 * to continue.
7135 */
7136 DERR("rc_conn_closed: timer is being processed\n");
7137 daplka_ep_set_state(ep_rp, old_state, new_state);
7138 return (IBT_CM_ACCEPT);
7139 }
7140
7141 /*
7142 * create a DAPL_IB_CME_DISCONNECTED event
7143 */
7144 disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
7145 if (disc_ev == NULL) {
7146 DERR("rc_conn_closed: could not alloc ev\n");
7147 daplka_ep_set_state(ep_rp, old_state, new_state);
7148 return (IBT_CM_ACCEPT);
7149 }
7150
7151 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_DISCONNECTED;
7152 disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
7153 disc_ev->ee_cmev.ec_cm_is_passive = B_FALSE;
7154 disc_ev->ee_cmev.ec_cm_psep_cookie = 0;
7155 disc_ev->ee_cmev.ec_cm_ev_priv_data = NULL;
7156 disc_ev->ee_cmev.ec_cm_ev_priv_data_len = 0;
7157
7158 D2("rc_conn_closed: enqueue event(%p) evdp(%p) closed(%d)\n",
7159 disc_ev, ep_rp->ep_conn_evd, (int)event->cm_event.closed);
7160
7161 /*
7162 * transition ep_state to DISCONNECTED
7163 */
7164 new_state = DAPLKA_EP_STATE_DISCONNECTED;
7165 daplka_ep_set_state(ep_rp, old_state, new_state);
7166
7167 /*
7168 * enqueue event onto the conn_evd owned by ep_rp
7169 */
7170 daplka_evd_wakeup(ep_rp->ep_conn_evd,
7171 &ep_rp->ep_conn_evd->evd_conn_events, disc_ev);
7172
7173 return (IBT_CM_ACCEPT);
7174 }
7175
7176 /*
7177 * processes the CONN_EST event
7178 */
7179 /* ARGSUSED */
7180 static ibt_cm_status_t
7181 daplka_cm_rc_conn_est(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
7182 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
7183 {
7184 daplka_evd_event_t *conn_ev;
7185 uint32_t old_state, new_state;
7186
7187 ASSERT(ep_rp != NULL);
7188 old_state = new_state = daplka_ep_get_state(ep_rp);
7189 if (old_state != DAPLKA_EP_STATE_CONNECTING) {
7190 /*
7191 * we can get here if the connection is being aborted
7192 */
7193 DERR("rc_conn_est: conn aborted, state = %d\n", old_state);
7194 daplka_ep_set_state(ep_rp, old_state, new_state);
7195 return (IBT_CM_ACCEPT);
7196 }
7197
7198 /*
7199 * it's ok for the timer to fire at this point. the
7200 * taskq thread that processes the timer will just wait
7201 * until we are done with our state transition.
7202 */
7203 if (daplka_cancel_timer(ep_rp) != 0) {
7204 /*
7205 * daplka_cancel_timer returns -1 if the timer is
7206 * being processed and 0 for all other cases.
7207 * we need to reset ep_state to allow timer processing
7208 * to continue.
7209 */
7210 DERR("rc_conn_est: timer is being processed\n");
7211 daplka_ep_set_state(ep_rp, old_state, new_state);
7212 return (IBT_CM_ACCEPT);
7213 }
7214
7215 /*
7216 * create a DAPL_IB_CME_CONNECTED event
7217 */
7218 conn_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
7219 if (conn_ev == NULL) {
7220 DERR("rc_conn_est: could not alloc ev\n");
7221 daplka_ep_set_state(ep_rp, old_state, new_state);
7222 return (IBT_CM_ACCEPT);
7223 }
7224
7225 conn_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_CONNECTED;
7226 conn_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
7227 conn_ev->ee_cmev.ec_cm_is_passive = B_FALSE;
7228 conn_ev->ee_cmev.ec_cm_psep_cookie = 0;
7229
7230 /*
7231 * The private data passed back in the connection established
7232 * event is what was recvd in the daplka_cm_rc_rep_rcv handler and
7233 * saved in ep resource structure.
7234 */
7235 if (ep_rp->ep_priv_len > 0) {
7236 conn_ev->ee_cmev.ec_cm_ev_priv_data =
7237 kmem_zalloc(ep_rp->ep_priv_len, KM_NOSLEEP);
7238
7239 if (conn_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
7240 DERR("rc_conn_est: could not alloc pr_data\n");
7241 kmem_free(conn_ev, sizeof (daplka_evd_event_t));
7242 daplka_ep_set_state(ep_rp, old_state, new_state);
7243 return (IBT_CM_ACCEPT);
7244 }
7245 bcopy(ep_rp->ep_priv_data, conn_ev->ee_cmev.ec_cm_ev_priv_data,
7246 ep_rp->ep_priv_len);
7247 }
7248 conn_ev->ee_cmev.ec_cm_ev_priv_data_len = ep_rp->ep_priv_len;
7249
7250 D2("rc_conn_est: enqueue event(%p) evdp(%p) pr_data(0x%p), "
7251 "pr_len(%d)\n", conn_ev, ep_rp->ep_conn_evd,
7252 conn_ev->ee_cmev.ec_cm_ev_priv_data,
7253 (int)conn_ev->ee_cmev.ec_cm_ev_priv_data_len);
7254
7255 /*
7256 * transition ep_state to CONNECTED
7257 */
7258 new_state = DAPLKA_EP_STATE_CONNECTED;
7259 daplka_ep_set_state(ep_rp, old_state, new_state);
7260
7261 /*
7262 * enqueue event onto the conn_evd owned by ep_rp
7263 */
7264 daplka_evd_wakeup(ep_rp->ep_conn_evd,
7265 &ep_rp->ep_conn_evd->evd_conn_events, conn_ev);
7266
7267 return (IBT_CM_ACCEPT);
7268 }
7269
7270 /*
7271 * processes the FAILURE event
7272 */
7273 /* ARGSUSED */
7274 static ibt_cm_status_t
7275 daplka_cm_rc_event_failure(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
7276 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
7277 {
7278 daplka_evd_event_t *disc_ev;
7279 ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
7280 void *pr_data = event->cm_priv_data;
7281 uint32_t old_state, new_state;
7282 ibt_rc_chan_query_attr_t chan_attrs;
7283 ibt_status_t status;
7284
7285 ASSERT(ep_rp != NULL);
7286 old_state = new_state = daplka_ep_get_state(ep_rp);
7287 if (old_state != DAPLKA_EP_STATE_CONNECTING &&
7288 old_state != DAPLKA_EP_STATE_DISCONNECTING) {
7289 /*
7290 * we can get here if the connection is being aborted
7291 */
7292 DERR("rc_event_failure: conn aborted, state = %d, "
7293 "cf_code = %d, cf_msg = %d, cf_reason = %d\n", old_state,
7294 (int)event->cm_event.failed.cf_code,
7295 (int)event->cm_event.failed.cf_msg,
7296 (int)event->cm_event.failed.cf_reason);
7297
7298 daplka_ep_set_state(ep_rp, old_state, new_state);
7299 return (IBT_CM_ACCEPT);
7300 }
7301
7302 /*
7303 * it's ok for the timer to fire at this point. the
7304 * taskq thread that processes the timer will just wait
7305 * until we are done with our state transition.
7306 */
7307 if (daplka_cancel_timer(ep_rp) != 0) {
7308 /*
7309 * daplka_cancel_timer returns -1 if the timer is
7310 * being processed and 0 for all other cases.
7311 * we need to reset ep_state to allow timer processing
7312 * to continue.
7313 */
7314 DERR("rc_event_failure: timer is being processed\n");
7315 daplka_ep_set_state(ep_rp, old_state, new_state);
7316 return (IBT_CM_ACCEPT);
7317 }
7318
7319 bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
7320 status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
7321
7322 if ((status == IBT_SUCCESS) &&
7323 (chan_attrs.rc_state != IBT_STATE_ERROR)) {
7324 DERR("rc_event_failure: conn abort qpn %d state %d\n",
7325 chan_attrs.rc_qpn, chan_attrs.rc_state);
7326
7327 /* explicit transition the QP to ERROR state */
7328 status = ibt_flush_channel(ep_rp->ep_chan_hdl);
7329 }
7330
7331 /*
7332 * create an event
7333 */
7334 disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
7335 if (disc_ev == NULL) {
7336 DERR("rc_event_failure: cannot alloc disc_ev\n");
7337 daplka_ep_set_state(ep_rp, old_state, new_state);
7338 return (IBT_CM_ACCEPT);
7339 }
7340
7341 /*
7342 * copy private data into event
7343 */
7344 if (pr_len > 0) {
7345 disc_ev->ee_cmev.ec_cm_ev_priv_data =
7346 kmem_zalloc(pr_len, KM_NOSLEEP);
7347
7348 if (disc_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
7349 DERR("rc_event_failure: cannot alloc pr data\n");
7350 kmem_free(disc_ev, sizeof (daplka_evd_event_t));
7351 daplka_ep_set_state(ep_rp, old_state, new_state);
7352 return (IBT_CM_ACCEPT);
7353 }
7354 bcopy(pr_data, disc_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
7355 }
7356 disc_ev->ee_cmev.ec_cm_ev_priv_data_len = pr_len;
7357
7358 /*
7359 * fill in the appropriate event type
7360 */
7361 if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_REJ_RCV) {
7362 switch (event->cm_event.failed.cf_reason) {
7363 case IBT_CM_CONSUMER:
7364 disc_ev->ee_cmev.ec_cm_ev_type =
7365 DAPL_IB_CME_DESTINATION_REJECT_PRIVATE_DATA;
7366 break;
7367 case IBT_CM_NO_CHAN:
7368 case IBT_CM_NO_RESC:
7369 disc_ev->ee_cmev.ec_cm_ev_type =
7370 DAPL_IB_CME_DESTINATION_REJECT;
7371 break;
7372 default:
7373 disc_ev->ee_cmev.ec_cm_ev_type =
7374 DAPL_IB_CME_DESTINATION_REJECT;
7375 break;
7376 }
7377 } else if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_TIMEOUT) {
7378 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
7379 } else {
7380 /* others we'll mark as local failure */
7381 disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_LOCAL_FAILURE;
7382 }
7383 disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
7384 disc_ev->ee_cmev.ec_cm_is_passive = B_FALSE;
7385 disc_ev->ee_cmev.ec_cm_psep_cookie = 0;
7386
7387 D2("rc_event_failure: enqueue event(%p) evdp(%p) cf_code(%d) "
7388 "cf_msg(%d) cf_reason(%d)\n", disc_ev, ep_rp->ep_conn_evd,
7389 (int)event->cm_event.failed.cf_code,
7390 (int)event->cm_event.failed.cf_msg,
7391 (int)event->cm_event.failed.cf_reason);
7392
7393 /*
7394 * transition ep_state to DISCONNECTED
7395 */
7396 new_state = DAPLKA_EP_STATE_DISCONNECTED;
7397 daplka_ep_set_state(ep_rp, old_state, new_state);
7398
7399 /*
7400 * enqueue event onto the conn_evd owned by ep_rp
7401 */
7402 daplka_evd_wakeup(ep_rp->ep_conn_evd,
7403 &ep_rp->ep_conn_evd->evd_conn_events, disc_ev);
7404
7405 return (IBT_CM_ACCEPT);
7406 }
7407
7408 /*
7409 * This is the active side CM handler. It gets registered when
7410 * ibt_open_rc_channel is called.
7411 */
7412 static ibt_cm_status_t
7413 daplka_cm_rc_handler(void *cm_private, ibt_cm_event_t *event,
7414 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
7415 {
7416 daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)cm_private;
7417
7418 if (ep_rp == NULL) {
7419 DERR("rc_handler: ep_rp == NULL\n");
7420 return (IBT_CM_NO_CHANNEL);
7421 }
7422 /*
7423 * default is not to return priv data
7424 */
7425 if (ret_args != NULL) {
7426 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ret_args))
7427 ret_args->cm_ret_len = 0;
7428 }
7429
7430 switch (event->cm_type) {
7431 case IBT_CM_EVENT_REQ_RCV:
7432 /* active side should not receive this event */
7433 D2("rc_handler: IBT_CM_EVENT_REQ_RCV\n");
7434 break;
7435
7436 case IBT_CM_EVENT_REP_RCV:
7437 /* connection accepted by passive side */
7438 D2("rc_handler: IBT_CM_EVENT_REP_RCV\n");
7439 return (daplka_cm_rc_rep_rcv(ep_rp, event, ret_args,
7440 priv_data, len));
7441
7442 case IBT_CM_EVENT_CONN_CLOSED:
7443 D2("rc_handler: IBT_CM_EVENT_CONN_CLOSED %d\n",
7444 event->cm_event.closed);
7445 return (daplka_cm_rc_conn_closed(ep_rp, event, ret_args,
7446 priv_data, len));
7447
7448 case IBT_CM_EVENT_MRA_RCV:
7449 /* passive side does default processing MRA event */
7450 D2("rc_handler: IBT_CM_EVENT_MRA_RCV\n");
7451 return (IBT_CM_DEFAULT);
7452
7453 case IBT_CM_EVENT_CONN_EST:
7454 D2("rc_handler: IBT_CM_EVENT_CONN_EST\n");
7455 return (daplka_cm_rc_conn_est(ep_rp, event, ret_args,
7456 priv_data, len));
7457
7458 case IBT_CM_EVENT_FAILURE:
7459 D2("rc_handler: IBT_CM_EVENT_FAILURE\n");
7460 return (daplka_cm_rc_event_failure(ep_rp, event, ret_args,
7461 priv_data, len));
7462
7463 default:
7464 D2("rc_handler: invalid event %d\n", event->cm_type);
7465 break;
7466 }
7467 return (IBT_CM_DEFAULT);
7468 }
7469
7470 /*
7471 * creates an IA resource and inserts it into the global resource table.
7472 */
7473 /* ARGSUSED */
7474 static int
7475 daplka_ia_create(minor_t rnum, intptr_t arg, int mode,
7476 cred_t *cred, int *rvalp)
7477 {
7478 daplka_ia_resource_t *ia_rp, *tmp_rp;
7479 boolean_t inserted = B_FALSE;
7480 dapl_ia_create_t args;
7481 ibt_hca_hdl_t hca_hdl;
7482 ibt_status_t status;
7483 ib_gid_t sgid;
7484 int retval;
7485 ibt_hca_portinfo_t *pinfop;
7486 uint_t pinfon;
7487 uint_t size;
7488 ibt_ar_t ar_s;
7489 daplka_hca_t *hca;
7490
7491 retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ia_create_t),
7492 mode);
7493 if (retval != 0) {
7494 DERR("ia_create: copyin error %d\n", retval);
7495 return (EFAULT);
7496 }
7497 if (args.ia_version != DAPL_IF_VERSION) {
7498 DERR("ia_create: invalid version %d, expected version %d\n",
7499 args.ia_version, DAPL_IF_VERSION);
7500 return (EINVAL);
7501 }
7502
7503 /*
7504 * find the hca with the matching guid
7505 */
7506 mutex_enter(&daplka_dev->daplka_mutex);
7507 for (hca = daplka_dev->daplka_hca_list_head; hca != NULL;
7508 hca = hca->hca_next) {
7509 if (hca->hca_guid == args.ia_guid) {
7510 DAPLKA_HOLD_HCA_WITHOUT_LOCK(hca);
7511 break;
7512 }
7513 }
7514 mutex_exit(&daplka_dev->daplka_mutex);
7515
7516 if (hca == NULL) {
7517 DERR("ia_create: guid 0x%016llx not found\n",
7518 (longlong_t)args.ia_guid);
7519 return (EINVAL);
7520 }
7521
7522 /*
7523 * check whether port number is valid and whether it is up
7524 */
7525 if (args.ia_port > hca->hca_nports) {
7526 DERR("ia_create: invalid hca_port %d\n", args.ia_port);
7527 DAPLKA_RELE_HCA(daplka_dev, hca);
7528 return (EINVAL);
7529 }
7530 hca_hdl = hca->hca_hdl;
7531 if (hca_hdl == NULL) {
7532 DERR("ia_create: hca_hdl == NULL\n");
7533 DAPLKA_RELE_HCA(daplka_dev, hca);
7534 return (EINVAL);
7535 }
7536 status = ibt_query_hca_ports(hca_hdl, (uint8_t)args.ia_port,
7537 &pinfop, &pinfon, &size);
7538 if (status != IBT_SUCCESS) {
7539 DERR("ia_create: ibt_query_hca_ports returned %d\n", status);
7540 *rvalp = (int)status;
7541 DAPLKA_RELE_HCA(daplka_dev, hca);
7542 return (0);
7543 }
7544 sgid = pinfop->p_sgid_tbl[0];
7545 ibt_free_portinfo(pinfop, size);
7546
7547 ia_rp = kmem_zalloc(sizeof (daplka_ia_resource_t), daplka_km_flags);
7548 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ia_rp))
7549 DAPLKA_RS_INIT(ia_rp, DAPL_TYPE_IA, rnum, daplka_ia_destroy);
7550
7551 mutex_init(&ia_rp->ia_lock, NULL, MUTEX_DRIVER, NULL);
7552 cv_init(&ia_rp->ia_cv, NULL, CV_DRIVER, NULL);
7553 ia_rp->ia_hca_hdl = hca_hdl;
7554 ia_rp->ia_hca_sgid = sgid;
7555 ia_rp->ia_hca = hca;
7556 ia_rp->ia_port_num = args.ia_port;
7557 ia_rp->ia_port_pkey = args.ia_pkey;
7558 ia_rp->ia_pid = ddi_get_pid();
7559 ia_rp->ia_async_evd_hkeys = NULL;
7560 ia_rp->ia_ar_registered = B_FALSE;
7561 bcopy(args.ia_sadata, ia_rp->ia_sadata, DAPL_ATS_NBYTES);
7562
7563 /* register Address Record */
7564 ar_s.ar_gid = ia_rp->ia_hca_sgid;
7565 ar_s.ar_pkey = ia_rp->ia_port_pkey;
7566 bcopy(ia_rp->ia_sadata, ar_s.ar_data, DAPL_ATS_NBYTES);
7567 #define UC(b) ar_s.ar_data[(b)]
7568 D3("daplka_ia_create: SA[8] %d.%d.%d.%d\n",
7569 UC(8), UC(9), UC(10), UC(11));
7570 D3("daplka_ia_create: SA[12] %d.%d.%d.%d\n",
7571 UC(12), UC(13), UC(14), UC(15));
7572 retval = ibt_register_ar(daplka_dev->daplka_clnt_hdl, &ar_s);
7573 if (retval != IBT_SUCCESS) {
7574 DERR("ia_create: failed to register Address Record.\n");
7575 retval = EINVAL;
7576 goto cleanup;
7577 }
7578 ia_rp->ia_ar_registered = B_TRUE;
7579
7580 /*
7581 * create hash tables for all object types
7582 */
7583 retval = daplka_hash_create(&ia_rp->ia_ep_htbl, DAPLKA_EP_HTBL_SZ,
7584 daplka_hash_ep_free, daplka_hash_generic_lookup);
7585 if (retval != 0) {
7586 DERR("ia_create: cannot create ep hash table\n");
7587 goto cleanup;
7588 }
7589 retval = daplka_hash_create(&ia_rp->ia_mr_htbl, DAPLKA_MR_HTBL_SZ,
7590 daplka_hash_mr_free, daplka_hash_generic_lookup);
7591 if (retval != 0) {
7592 DERR("ia_create: cannot create mr hash table\n");
7593 goto cleanup;
7594 }
7595 retval = daplka_hash_create(&ia_rp->ia_mw_htbl, DAPLKA_MW_HTBL_SZ,
7596 daplka_hash_mw_free, daplka_hash_generic_lookup);
7597 if (retval != 0) {
7598 DERR("ia_create: cannot create mw hash table\n");
7599 goto cleanup;
7600 }
7601 retval = daplka_hash_create(&ia_rp->ia_pd_htbl, DAPLKA_PD_HTBL_SZ,
7602 daplka_hash_pd_free, daplka_hash_generic_lookup);
7603 if (retval != 0) {
7604 DERR("ia_create: cannot create pd hash table\n");
7605 goto cleanup;
7606 }
7607 retval = daplka_hash_create(&ia_rp->ia_evd_htbl, DAPLKA_EVD_HTBL_SZ,
7608 daplka_hash_evd_free, daplka_hash_generic_lookup);
7609 if (retval != 0) {
7610 DERR("ia_create: cannot create evd hash table\n");
7611 goto cleanup;
7612 }
7613 retval = daplka_hash_create(&ia_rp->ia_cno_htbl, DAPLKA_CNO_HTBL_SZ,
7614 daplka_hash_cno_free, daplka_hash_generic_lookup);
7615 if (retval != 0) {
7616 DERR("ia_create: cannot create cno hash table\n");
7617 goto cleanup;
7618 }
7619 retval = daplka_hash_create(&ia_rp->ia_sp_htbl, DAPLKA_SP_HTBL_SZ,
7620 daplka_hash_sp_free, daplka_hash_generic_lookup);
7621 if (retval != 0) {
7622 DERR("ia_create: cannot create sp hash table\n");
7623 goto cleanup;
7624 }
7625 retval = daplka_hash_create(&ia_rp->ia_srq_htbl, DAPLKA_SRQ_HTBL_SZ,
7626 daplka_hash_srq_free, daplka_hash_generic_lookup);
7627 if (retval != 0) {
7628 DERR("ia_create: cannot create srq hash table\n");
7629 goto cleanup;
7630 }
7631 /*
7632 * insert ia_rp into the global resource table
7633 */
7634 retval = daplka_resource_insert(rnum, (daplka_resource_t *)ia_rp);
7635 if (retval != 0) {
7636 DERR("ia_create: cannot insert resource\n");
7637 goto cleanup;
7638 }
7639 inserted = B_TRUE;
7640 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ia_rp))
7641
7642 args.ia_resnum = rnum;
7643 retval = copyout(&args, (void *)arg, sizeof (dapl_ia_create_t));
7644 if (retval != 0) {
7645 DERR("ia_create: copyout error %d\n", retval);
7646 retval = EFAULT;
7647 goto cleanup;
7648 }
7649 return (0);
7650
7651 cleanup:;
7652 if (inserted) {
7653 tmp_rp = (daplka_ia_resource_t *)daplka_resource_remove(rnum);
7654 if (tmp_rp != ia_rp) {
7655 /*
7656 * we can return here because another thread must
7657 * have freed up the resource
7658 */
7659 DERR("ia_create: cannot remove resource\n");
7660 return (retval);
7661 }
7662 }
7663 DAPLKA_RS_UNREF(ia_rp);
7664 return (retval);
7665 }
7666
7667 /*
7668 * destroys an IA resource
7669 */
7670 static int
7671 daplka_ia_destroy(daplka_resource_t *gen_rp)
7672 {
7673 daplka_ia_resource_t *ia_rp = (daplka_ia_resource_t *)gen_rp;
7674 daplka_async_evd_hkey_t *hkp;
7675 int cnt;
7676 ibt_ar_t ar_s;
7677
7678 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ia_rp))
7679 D3("ia_destroy: entering, ia_rp 0x%p\n", ia_rp);
7680
7681 /* deregister Address Record */
7682 if (ia_rp->ia_ar_registered) {
7683 ar_s.ar_gid = ia_rp->ia_hca_sgid;
7684 ar_s.ar_pkey = ia_rp->ia_port_pkey;
7685 bcopy(ia_rp->ia_sadata, ar_s.ar_data, DAPL_ATS_NBYTES);
7686 (void) ibt_deregister_ar(daplka_dev->daplka_clnt_hdl, &ar_s);
7687 ia_rp->ia_ar_registered = B_FALSE;
7688 }
7689
7690 /*
7691 * destroy hash tables. make sure resources are
7692 * destroyed in the correct order.
7693 */
7694 daplka_hash_destroy(&ia_rp->ia_mw_htbl);
7695 daplka_hash_destroy(&ia_rp->ia_mr_htbl);
7696 daplka_hash_destroy(&ia_rp->ia_ep_htbl);
7697 daplka_hash_destroy(&ia_rp->ia_srq_htbl);
7698 daplka_hash_destroy(&ia_rp->ia_evd_htbl);
7699 daplka_hash_destroy(&ia_rp->ia_cno_htbl);
7700 daplka_hash_destroy(&ia_rp->ia_pd_htbl);
7701 daplka_hash_destroy(&ia_rp->ia_sp_htbl);
7702
7703 /*
7704 * free the async evd list
7705 */
7706 cnt = 0;
7707 hkp = ia_rp->ia_async_evd_hkeys;
7708 while (hkp != NULL) {
7709 daplka_async_evd_hkey_t *free_hkp;
7710
7711 cnt++;
7712 free_hkp = hkp;
7713 hkp = hkp->aeh_next;
7714 kmem_free(free_hkp, sizeof (*free_hkp));
7715 }
7716 if (cnt > 0) {
7717 D3("ia_destroy: freed %d hkeys\n", cnt);
7718 }
7719 mutex_destroy(&ia_rp->ia_lock);
7720 cv_destroy(&ia_rp->ia_cv);
7721 ia_rp->ia_hca_hdl = NULL;
7722
7723 DAPLKA_RS_FINI(ia_rp);
7724
7725 if (ia_rp->ia_hca)
7726 DAPLKA_RELE_HCA(daplka_dev, ia_rp->ia_hca);
7727
7728 kmem_free(ia_rp, sizeof (daplka_ia_resource_t));
7729 D3("ia_destroy: exiting, ia_rp 0x%p\n", ia_rp);
7730 return (0);
7731 }
7732
7733 static void
7734 daplka_async_event_create(ibt_async_code_t code, ibt_async_event_t *event,
7735 uint64_t cookie, daplka_ia_resource_t *ia_rp)
7736 {
7737 daplka_evd_event_t *evp;
7738 daplka_evd_resource_t *async_evd;
7739 daplka_async_evd_hkey_t *curr;
7740
7741 mutex_enter(&ia_rp->ia_lock);
7742 curr = ia_rp->ia_async_evd_hkeys;
7743 while (curr != NULL) {
7744 /*
7745 * Note: this allocation does not zero out the buffer
7746 * since we init all the fields.
7747 */
7748 evp = kmem_alloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
7749 if (evp == NULL) {
7750 DERR("async_event_enqueue: event alloc failed"
7751 "!found\n", ia_rp, curr->aeh_evd_hkey);
7752 curr = curr->aeh_next;
7753 continue;
7754 }
7755 evp->ee_next = NULL;
7756 evp->ee_aev.ibae_type = code;
7757 evp->ee_aev.ibae_hca_guid = event->ev_hca_guid;
7758 evp->ee_aev.ibae_cookie = cookie;
7759 evp->ee_aev.ibae_port = event->ev_port;
7760
7761 /*
7762 * Lookup the async evd corresponding to this ia and enqueue
7763 * evp and wakeup any waiter.
7764 */
7765 async_evd = (daplka_evd_resource_t *)
7766 daplka_hash_lookup(&ia_rp->ia_evd_htbl, curr->aeh_evd_hkey);
7767 if (async_evd == NULL) { /* async evd is being freed */
7768 DERR("async_event_enqueue: ia_rp(%p) asycn_evd %llx "
7769 "!found\n", ia_rp, (longlong_t)curr->aeh_evd_hkey);
7770 kmem_free(evp, sizeof (daplka_evd_event_t));
7771 curr = curr->aeh_next;
7772 continue;
7773 }
7774 daplka_evd_wakeup(async_evd, &async_evd->evd_async_events, evp);
7775
7776 /* decrement refcnt on async_evd */
7777 DAPLKA_RS_UNREF(async_evd);
7778 curr = curr->aeh_next;
7779 }
7780 mutex_exit(&ia_rp->ia_lock);
7781 }
7782 /*
7783 * This routine is called in kernel context
7784 */
7785
7786 /* ARGSUSED */
7787 static void
7788 daplka_rc_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
7789 ibt_async_code_t code, ibt_async_event_t *event)
7790 {
7791 daplka_ep_resource_t *epp;
7792 daplka_ia_resource_t *ia_rp;
7793 minor_t ia_rnum;
7794
7795 if (event->ev_chan_hdl == NULL) {
7796 DERR("daplka_rc_async_handler: ev_chan_hdl is NULL\n");
7797 return;
7798 }
7799
7800 mutex_enter(&daplka_dev->daplka_mutex);
7801 epp = ibt_get_chan_private(event->ev_chan_hdl);
7802 if (epp == NULL) {
7803 mutex_exit(&daplka_dev->daplka_mutex);
7804 DERR("daplka_rc_async_handler: chan_private is NULL\n");
7805 return;
7806 }
7807
7808 /* grab a reference to this ep */
7809 DAPLKA_RS_REF(epp);
7810 mutex_exit(&daplka_dev->daplka_mutex);
7811
7812 /*
7813 * The endpoint resource has the resource number corresponding to
7814 * the IA resource. Use that to lookup the ia resource entry
7815 */
7816 ia_rnum = DAPLKA_RS_RNUM(epp);
7817 ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(ia_rnum);
7818 if ((ia_rp == NULL) || DAPLKA_RS_RESERVED(ia_rp)) {
7819 D2("daplka_rc_async_handler: resource (%d) not found\n",
7820 ia_rnum);
7821 DAPLKA_RS_UNREF(epp);
7822 return;
7823 }
7824
7825 /*
7826 * Create an async event and chain it to the async evd
7827 */
7828 daplka_async_event_create(code, event, epp->ep_cookie, ia_rp);
7829
7830 DAPLKA_RS_UNREF(ia_rp);
7831 DAPLKA_RS_UNREF(epp);
7832 }
7833
7834 /*
7835 * This routine is called in kernel context
7836 */
7837
7838 /* ARGSUSED */
7839 static void
7840 daplka_cq_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
7841 ibt_async_code_t code, ibt_async_event_t *event)
7842 {
7843 daplka_evd_resource_t *evdp;
7844 daplka_ia_resource_t *ia_rp;
7845 minor_t ia_rnum;
7846
7847 if (event->ev_cq_hdl == NULL)
7848 return;
7849
7850 mutex_enter(&daplka_dev->daplka_mutex);
7851 evdp = ibt_get_cq_private(event->ev_cq_hdl);
7852 if (evdp == NULL) {
7853 mutex_exit(&daplka_dev->daplka_mutex);
7854 DERR("daplka_cq_async_handler: get cq private(%p) failed\n",
7855 event->ev_cq_hdl);
7856 return;
7857 }
7858 /* grab a reference to this evd resource */
7859 DAPLKA_RS_REF(evdp);
7860 mutex_exit(&daplka_dev->daplka_mutex);
7861
7862 /*
7863 * The endpoint resource has the resource number corresponding to
7864 * the IA resource. Use that to lookup the ia resource entry
7865 */
7866 ia_rnum = DAPLKA_RS_RNUM(evdp);
7867 ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(ia_rnum);
7868 if ((ia_rp == NULL) || DAPLKA_RS_RESERVED(ia_rp)) {
7869 DERR("daplka_cq_async_handler: resource (%d) not found\n",
7870 ia_rnum);
7871 DAPLKA_RS_UNREF(evdp);
7872 return;
7873 }
7874
7875 /*
7876 * Create an async event and chain it to the async evd
7877 */
7878 daplka_async_event_create(code, event, evdp->evd_cookie, ia_rp);
7879
7880 /* release all the refcount that were acquired */
7881 DAPLKA_RS_UNREF(ia_rp);
7882 DAPLKA_RS_UNREF(evdp);
7883 }
7884
7885 /*
7886 * This routine is called in kernel context, handles unaffiliated async errors
7887 */
7888
7889 /* ARGSUSED */
7890 static void
7891 daplka_un_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
7892 ibt_async_code_t code, ibt_async_event_t *event)
7893 {
7894 int i, j;
7895 daplka_resource_blk_t *blk;
7896 daplka_resource_t *rp;
7897 daplka_ia_resource_t *ia_rp;
7898
7899 /*
7900 * Walk the resource table looking for an ia that matches the
7901 * hca_hdl.
7902 */
7903 rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
7904 for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
7905 blk = daplka_resource.daplka_rc_root[i];
7906 if (blk == NULL)
7907 continue;
7908 for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
7909 rp = blk->daplka_rcblk_blks[j];
7910 if ((rp == NULL) ||
7911 ((intptr_t)rp == DAPLKA_RC_RESERVED) ||
7912 (rp->rs_type != DAPL_TYPE_IA)) {
7913 continue;
7914 }
7915 /*
7916 * rp is an IA resource check if it belongs
7917 * to the hca/port for which we got the event
7918 */
7919 ia_rp = (daplka_ia_resource_t *)rp;
7920 DAPLKA_RS_REF(ia_rp);
7921 if ((hca_hdl == ia_rp->ia_hca_hdl) &&
7922 (event->ev_port == ia_rp->ia_port_num)) {
7923 /*
7924 * walk the ep hash table. Acquire a
7925 * reader lock. NULL dgid indicates
7926 * local port up event.
7927 */
7928 daplka_hash_walk(&ia_rp->ia_ep_htbl,
7929 daplka_ep_failback, NULL, RW_READER);
7930 }
7931 DAPLKA_RS_UNREF(ia_rp);
7932 }
7933 }
7934 rw_exit(&daplka_resource.daplka_rct_lock);
7935 }
7936
7937 static int
7938 daplka_handle_hca_detach_event(ibt_async_event_t *event)
7939 {
7940 daplka_hca_t *hca;
7941
7942 /*
7943 * find the hca with the matching guid
7944 */
7945 mutex_enter(&daplka_dev->daplka_mutex);
7946 for (hca = daplka_dev->daplka_hca_list_head; hca != NULL;
7947 hca = hca->hca_next) {
7948 if (hca->hca_guid == event->ev_hca_guid) {
7949 if (DAPLKA_HCA_BUSY(hca)) {
7950 mutex_exit(&daplka_dev->daplka_mutex);
7951 return (IBT_HCA_RESOURCES_NOT_FREED);
7952 }
7953 daplka_dequeue_hca(daplka_dev, hca);
7954 break;
7955 }
7956 }
7957 mutex_exit(&daplka_dev->daplka_mutex);
7958
7959 if (hca == NULL)
7960 return (IBT_FAILURE);
7961
7962 return (daplka_fini_hca(daplka_dev, hca));
7963 }
7964
7965 /*
7966 * This routine is called in kernel context
7967 */
7968 static void
7969 daplka_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
7970 ibt_async_code_t code, ibt_async_event_t *event)
7971 {
7972 switch (code) {
7973 case IBT_ERROR_CATASTROPHIC_CHAN:
7974 case IBT_ERROR_INVALID_REQUEST_CHAN:
7975 case IBT_ERROR_ACCESS_VIOLATION_CHAN:
7976 case IBT_ERROR_PATH_MIGRATE_REQ:
7977 D2("daplka_async_handler(): Channel affiliated=0x%x\n", code);
7978 /* These events are affiliated with a the RC channel */
7979 daplka_rc_async_handler(clnt_private, hca_hdl, code, event);
7980 break;
7981 case IBT_ERROR_CQ:
7982 /* This event is affiliated with a the CQ */
7983 D2("daplka_async_handler(): IBT_ERROR_CQ\n");
7984 daplka_cq_async_handler(clnt_private, hca_hdl, code, event);
7985 break;
7986 case IBT_ERROR_PORT_DOWN:
7987 D2("daplka_async_handler(): IBT_PORT_DOWN\n");
7988 break;
7989 case IBT_EVENT_PORT_UP:
7990 D2("daplka_async_handler(): IBT_PORT_UP\n");
7991 if (daplka_apm) {
7992 daplka_un_async_handler(clnt_private, hca_hdl, code,
7993 event);
7994 }
7995 break;
7996 case IBT_HCA_ATTACH_EVENT:
7997 /*
7998 * NOTE: In some error recovery paths, it is possible to
7999 * receive IBT_HCA_ATTACH_EVENTs on already known HCAs.
8000 */
8001 D2("daplka_async_handler(): IBT_HCA_ATTACH\n");
8002 (void) daplka_init_hca(daplka_dev, event->ev_hca_guid);
8003 break;
8004 case IBT_HCA_DETACH_EVENT:
8005 D2("daplka_async_handler(): IBT_HCA_DETACH\n");
8006 /* Free all hca resources and close the HCA. */
8007 (void) daplka_handle_hca_detach_event(event);
8008 break;
8009 case IBT_EVENT_PATH_MIGRATED:
8010 /* This event is affiliated with APM */
8011 D2("daplka_async_handler(): IBT_PATH_MIGRATED.\n");
8012 break;
8013 default:
8014 D2("daplka_async_handler(): unhandled code = 0x%x\n", code);
8015 break;
8016 }
8017 }
8018
8019 /*
8020 * This routine is called in kernel context related to Subnet events
8021 */
8022 /*ARGSUSED*/
8023 static void
8024 daplka_sm_notice_handler(void *arg, ib_gid_t gid, ibt_subnet_event_code_t code,
8025 ibt_subnet_event_t *event)
8026 {
8027 ib_gid_t *sgid = &gid;
8028 ib_gid_t *dgid;
8029
8030 dgid = &event->sm_notice_gid;
8031 switch (code) {
8032 case IBT_SM_EVENT_GID_AVAIL:
8033 /* This event is affiliated with remote port up */
8034 D2("daplka_sm_notice_handler(): IBT_SM_EVENT_GID_AVAIL\n");
8035 if (daplka_apm)
8036 daplka_sm_gid_avail(sgid, dgid);
8037 return;
8038 case IBT_SM_EVENT_GID_UNAVAIL:
8039 /* This event is affiliated with remote port down */
8040 D2("daplka_sm_notice_handler(): IBT_SM_EVENT_GID_UNAVAIL\n");
8041 return;
8042 default:
8043 D2("daplka_sm_notice_handler(): unhandled IBT_SM_EVENT_[%d]\n",
8044 code);
8045 return;
8046 }
8047 }
8048
8049 /*
8050 * This routine is called in kernel context, handles Subnet GID avail events
8051 * which correspond to remote port up. Setting up alternate path or path
8052 * migration (failback) has to be initiated from the active side of the
8053 * original connect.
8054 */
8055 static void
8056 daplka_sm_gid_avail(ib_gid_t *sgid, ib_gid_t *dgid)
8057 {
8058 int i, j;
8059 daplka_resource_blk_t *blk;
8060 daplka_resource_t *rp;
8061 daplka_ia_resource_t *ia_rp;
8062
8063 D2("daplka_sm_gid_avail: sgid=%llx:%llx dgid=%llx:%llx\n",
8064 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid,
8065 (longlong_t)dgid->gid_prefix, (longlong_t)dgid->gid_guid);
8066
8067 /*
8068 * Walk the resource table looking for an ia that matches the sgid
8069 */
8070 rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
8071 for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
8072 blk = daplka_resource.daplka_rc_root[i];
8073 if (blk == NULL)
8074 continue;
8075 for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
8076 rp = blk->daplka_rcblk_blks[j];
8077 if ((rp == NULL) ||
8078 ((intptr_t)rp == DAPLKA_RC_RESERVED) ||
8079 (rp->rs_type != DAPL_TYPE_IA)) {
8080 continue;
8081 }
8082 /*
8083 * rp is an IA resource check if its gid
8084 * matches with the calling sgid
8085 */
8086 ia_rp = (daplka_ia_resource_t *)rp;
8087 DAPLKA_RS_REF(ia_rp);
8088 if ((sgid->gid_prefix ==
8089 ia_rp->ia_hca_sgid.gid_prefix) &&
8090 (sgid->gid_guid == ia_rp->ia_hca_sgid.gid_guid)) {
8091 /*
8092 * walk the ep hash table. Acquire a
8093 * reader lock.
8094 */
8095 daplka_hash_walk(&ia_rp->ia_ep_htbl,
8096 daplka_ep_failback,
8097 (void *)dgid, RW_READER);
8098 }
8099 DAPLKA_RS_UNREF(ia_rp);
8100 }
8101 }
8102 rw_exit(&daplka_resource.daplka_rct_lock);
8103 }
8104
8105 /*
8106 * This routine is called in kernel context to get and set an alternate path
8107 */
8108 static int
8109 daplka_ep_altpath(daplka_ep_resource_t *ep_rp, ib_gid_t *dgid)
8110 {
8111 ibt_alt_path_info_t path_info;
8112 ibt_alt_path_attr_t path_attr;
8113 ibt_ap_returns_t ap_rets;
8114 ibt_status_t status;
8115
8116 D2("daplka_ep_altpath : ibt_get_alt_path()\n");
8117 bzero(&path_info, sizeof (ibt_alt_path_info_t));
8118 bzero(&path_attr, sizeof (ibt_alt_path_attr_t));
8119 if (dgid != NULL) {
8120 path_attr.apa_sgid = ep_rp->ep_sgid;
8121 path_attr.apa_dgid = *dgid;
8122 }
8123 status = ibt_get_alt_path(ep_rp->ep_chan_hdl, IBT_PATH_AVAIL,
8124 &path_attr, &path_info);
8125 if (status != IBT_SUCCESS) {
8126 DERR("daplka_ep_altpath : ibt_get_alt_path failed %d\n",
8127 status);
8128 return (1);
8129 }
8130
8131 D2("daplka_ep_altpath : ibt_set_alt_path()\n");
8132 bzero(&ap_rets, sizeof (ibt_ap_returns_t));
8133 status = ibt_set_alt_path(ep_rp->ep_chan_hdl, IBT_BLOCKING,
8134 &path_info, NULL, 0, &ap_rets);
8135 if ((status != IBT_SUCCESS) ||
8136 (ap_rets.ap_status != IBT_CM_AP_LOADED)) {
8137 DERR("daplka_ep_altpath : ibt_set_alt_path failed "
8138 "status %d ap_status %d\n", status, ap_rets.ap_status);
8139 return (1);
8140 }
8141 return (0);
8142 }
8143
8144 /*
8145 * This routine is called in kernel context to failback to the original path
8146 */
8147 static int
8148 daplka_ep_failback(void *objp, void *arg)
8149 {
8150 daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)objp;
8151 ib_gid_t *dgid;
8152 ibt_status_t status;
8153 ibt_rc_chan_query_attr_t chan_attrs;
8154 int i;
8155
8156 ASSERT(DAPLKA_RS_TYPE(ep_rp) == DAPL_TYPE_EP);
8157 D2("daplka_ep_failback ep : sgid=%llx:%llx dgid=%llx:%llx\n",
8158 (longlong_t)ep_rp->ep_sgid.gid_prefix,
8159 (longlong_t)ep_rp->ep_sgid.gid_guid,
8160 (longlong_t)ep_rp->ep_dgid.gid_prefix,
8161 (longlong_t)ep_rp->ep_dgid.gid_guid);
8162
8163 /*
8164 * daplka_ep_failback is called from daplka_hash_walk
8165 * which holds the read lock on hash table to protect
8166 * the endpoint resource from removal
8167 */
8168 mutex_enter(&ep_rp->ep_lock);
8169 /* check for unconnected endpoints */
8170 /* first check for ep state */
8171 if (ep_rp->ep_state != DAPLKA_EP_STATE_CONNECTED) {
8172 mutex_exit(&ep_rp->ep_lock);
8173 D2("daplka_ep_failback : endpoints not connected\n");
8174 return (0);
8175 }
8176
8177 /* second check for gids */
8178 if (((ep_rp->ep_sgid.gid_prefix == 0) &&
8179 (ep_rp->ep_sgid.gid_guid == 0)) ||
8180 ((ep_rp->ep_dgid.gid_prefix == 0) &&
8181 (ep_rp->ep_dgid.gid_guid == 0))) {
8182 mutex_exit(&ep_rp->ep_lock);
8183 D2("daplka_ep_failback : skip unconnected endpoints\n");
8184 return (0);
8185 }
8186
8187 /*
8188 * matching destination ep
8189 * when dgid is NULL, the async event is a local port up.
8190 * dgid becomes wild card, i.e. all endpoints match
8191 */
8192 dgid = (ib_gid_t *)arg;
8193 if (dgid == NULL) {
8194 /* ignore loopback ep */
8195 if ((ep_rp->ep_sgid.gid_prefix == ep_rp->ep_dgid.gid_prefix) &&
8196 (ep_rp->ep_sgid.gid_guid == ep_rp->ep_dgid.gid_guid)) {
8197 mutex_exit(&ep_rp->ep_lock);
8198 D2("daplka_ep_failback : skip loopback endpoints\n");
8199 return (0);
8200 }
8201 } else {
8202 /* matching remote ep */
8203 if ((ep_rp->ep_dgid.gid_prefix != dgid->gid_prefix) ||
8204 (ep_rp->ep_dgid.gid_guid != dgid->gid_guid)) {
8205 mutex_exit(&ep_rp->ep_lock);
8206 D2("daplka_ep_failback : unrelated endpoints\n");
8207 return (0);
8208 }
8209 }
8210
8211 /* call get and set altpath with original dgid used in ep_connect */
8212 if (daplka_ep_altpath(ep_rp, &ep_rp->ep_dgid)) {
8213 mutex_exit(&ep_rp->ep_lock);
8214 return (0);
8215 }
8216
8217 /*
8218 * wait for migration state to be ARMed
8219 * e.g. a post_send msg will transit mig_state from REARM to ARM
8220 */
8221 for (i = 0; i < daplka_query_aft_setaltpath; i++) {
8222 bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
8223 status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
8224 if (status != IBT_SUCCESS) {
8225 mutex_exit(&ep_rp->ep_lock);
8226 DERR("daplka_ep_altpath : ibt_query_rc_channel err\n");
8227 return (0);
8228 }
8229 if (chan_attrs.rc_mig_state == IBT_STATE_ARMED)
8230 break;
8231 }
8232
8233 D2("daplka_ep_altpath : query[%d] mig_st=%d\n",
8234 i, chan_attrs.rc_mig_state);
8235 D2("daplka_ep_altpath : P sgid=%llx:%llx dgid=%llx:%llx\n",
8236 (longlong_t)
8237 chan_attrs.rc_prim_path.cep_adds_vect.av_sgid.gid_prefix,
8238 (longlong_t)chan_attrs.rc_prim_path.cep_adds_vect.av_sgid.gid_guid,
8239 (longlong_t)
8240 chan_attrs.rc_prim_path.cep_adds_vect.av_dgid.gid_prefix,
8241 (longlong_t)chan_attrs.rc_prim_path.cep_adds_vect.av_dgid.gid_guid);
8242 D2("daplka_ep_altpath : A sgid=%llx:%llx dgid=%llx:%llx\n",
8243 (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_sgid.gid_prefix,
8244 (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_sgid.gid_guid,
8245 (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_dgid.gid_prefix,
8246 (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_dgid.gid_guid);
8247
8248 /* skip failback on ARMed state not reached or env override */
8249 if ((i >= daplka_query_aft_setaltpath) || (daplka_failback == 0)) {
8250 mutex_exit(&ep_rp->ep_lock);
8251 DERR("daplka_ep_altpath : ARMed state not reached\n");
8252 return (0);
8253 }
8254
8255 D2("daplka_ep_failback : ibt_migrate_path() to original ep\n");
8256 status = ibt_migrate_path(ep_rp->ep_chan_hdl);
8257 if (status != IBT_SUCCESS) {
8258 mutex_exit(&ep_rp->ep_lock);
8259 DERR("daplka_ep_failback : migration failed "
8260 "status %d\n", status);
8261 return (0);
8262 }
8263
8264 /* call get and altpath with NULL dgid to indicate unspecified dgid */
8265 (void) daplka_ep_altpath(ep_rp, NULL);
8266 mutex_exit(&ep_rp->ep_lock);
8267 return (0);
8268 }
8269
8270 /*
8271 * IBTF wrappers used for resource accounting
8272 */
8273 static ibt_status_t
8274 daplka_ibt_alloc_rc_channel(daplka_ep_resource_t *ep_rp, ibt_hca_hdl_t hca_hdl,
8275 ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args,
8276 ibt_channel_hdl_t *chan_hdl_p, ibt_chan_sizes_t *sizes)
8277 {
8278 daplka_hca_t *hca_p;
8279 uint32_t max_qps;
8280 boolean_t acct_enabled;
8281 ibt_status_t status;
8282
8283 acct_enabled = daplka_accounting_enabled;
8284 hca_p = ep_rp->ep_hca;
8285 max_qps = daplka_max_qp_percent * hca_p->hca_attr.hca_max_chans / 100;
8286
8287 if (acct_enabled) {
8288 if (daplka_max_qp_percent != 0 &&
8289 max_qps <= hca_p->hca_qp_count) {
8290 DERR("ibt_alloc_rc_channel: resource limit exceeded "
8291 "(limit %d, count %d)\n", max_qps,
8292 hca_p->hca_qp_count);
8293 return (IBT_INSUFF_RESOURCE);
8294 }
8295 DAPLKA_RS_ACCT_INC(ep_rp, 1);
8296 atomic_inc_32(&hca_p->hca_qp_count);
8297 }
8298 status = ibt_alloc_rc_channel(hca_hdl, flags, args, chan_hdl_p, sizes);
8299
8300 if (status != IBT_SUCCESS && acct_enabled) {
8301 DAPLKA_RS_ACCT_DEC(ep_rp, 1);
8302 atomic_dec_32(&hca_p->hca_qp_count);
8303 }
8304 return (status);
8305 }
8306
8307 static ibt_status_t
8308 daplka_ibt_free_channel(daplka_ep_resource_t *ep_rp, ibt_channel_hdl_t chan_hdl)
8309 {
8310 daplka_hca_t *hca_p;
8311 ibt_status_t status;
8312
8313 hca_p = ep_rp->ep_hca;
8314
8315 status = ibt_free_channel(chan_hdl);
8316 if (status != IBT_SUCCESS) {
8317 return (status);
8318 }
8319 if (DAPLKA_RS_ACCT_CHARGED(ep_rp) > 0) {
8320 DAPLKA_RS_ACCT_DEC(ep_rp, 1);
8321 atomic_dec_32(&hca_p->hca_qp_count);
8322 }
8323 return (status);
8324 }
8325
8326 static ibt_status_t
8327 daplka_ibt_alloc_cq(daplka_evd_resource_t *evd_rp, ibt_hca_hdl_t hca_hdl,
8328 ibt_cq_attr_t *cq_attr, ibt_cq_hdl_t *ibt_cq_p, uint32_t *real_size)
8329 {
8330 daplka_hca_t *hca_p;
8331 uint32_t max_cqs;
8332 boolean_t acct_enabled;
8333 ibt_status_t status;
8334
8335 acct_enabled = daplka_accounting_enabled;
8336 hca_p = evd_rp->evd_hca;
8337 max_cqs = daplka_max_cq_percent * hca_p->hca_attr.hca_max_cq / 100;
8338
8339 if (acct_enabled) {
8340 if (daplka_max_cq_percent != 0 &&
8341 max_cqs <= hca_p->hca_cq_count) {
8342 DERR("ibt_alloc_cq: resource limit exceeded "
8343 "(limit %d, count %d)\n", max_cqs,
8344 hca_p->hca_cq_count);
8345 return (IBT_INSUFF_RESOURCE);
8346 }
8347 DAPLKA_RS_ACCT_INC(evd_rp, 1);
8348 atomic_inc_32(&hca_p->hca_cq_count);
8349 }
8350 status = ibt_alloc_cq(hca_hdl, cq_attr, ibt_cq_p, real_size);
8351
8352 if (status != IBT_SUCCESS && acct_enabled) {
8353 DAPLKA_RS_ACCT_DEC(evd_rp, 1);
8354 atomic_dec_32(&hca_p->hca_cq_count);
8355 }
8356 return (status);
8357 }
8358
8359 static ibt_status_t
8360 daplka_ibt_free_cq(daplka_evd_resource_t *evd_rp, ibt_cq_hdl_t cq_hdl)
8361 {
8362 daplka_hca_t *hca_p;
8363 ibt_status_t status;
8364
8365 hca_p = evd_rp->evd_hca;
8366
8367 status = ibt_free_cq(cq_hdl);
8368 if (status != IBT_SUCCESS) {
8369 return (status);
8370 }
8371 if (DAPLKA_RS_ACCT_CHARGED(evd_rp) > 0) {
8372 DAPLKA_RS_ACCT_DEC(evd_rp, 1);
8373 atomic_dec_32(&hca_p->hca_cq_count);
8374 }
8375 return (status);
8376 }
8377
8378 static ibt_status_t
8379 daplka_ibt_alloc_pd(daplka_pd_resource_t *pd_rp, ibt_hca_hdl_t hca_hdl,
8380 ibt_pd_flags_t flags, ibt_pd_hdl_t *pd_hdl_p)
8381 {
8382 daplka_hca_t *hca_p;
8383 uint32_t max_pds;
8384 boolean_t acct_enabled;
8385 ibt_status_t status;
8386
8387 acct_enabled = daplka_accounting_enabled;
8388 hca_p = pd_rp->pd_hca;
8389 max_pds = daplka_max_pd_percent * hca_p->hca_attr.hca_max_pd / 100;
8390
8391 if (acct_enabled) {
8392 if (daplka_max_pd_percent != 0 &&
8393 max_pds <= hca_p->hca_pd_count) {
8394 DERR("ibt_alloc_pd: resource limit exceeded "
8395 "(limit %d, count %d)\n", max_pds,
8396 hca_p->hca_pd_count);
8397 return (IBT_INSUFF_RESOURCE);
8398 }
8399 DAPLKA_RS_ACCT_INC(pd_rp, 1);
8400 atomic_inc_32(&hca_p->hca_pd_count);
8401 }
8402 status = ibt_alloc_pd(hca_hdl, flags, pd_hdl_p);
8403
8404 if (status != IBT_SUCCESS && acct_enabled) {
8405 DAPLKA_RS_ACCT_DEC(pd_rp, 1);
8406 atomic_dec_32(&hca_p->hca_pd_count);
8407 }
8408 return (status);
8409 }
8410
8411 static ibt_status_t
8412 daplka_ibt_free_pd(daplka_pd_resource_t *pd_rp, ibt_hca_hdl_t hca_hdl,
8413 ibt_pd_hdl_t pd_hdl)
8414 {
8415 daplka_hca_t *hca_p;
8416 ibt_status_t status;
8417
8418 hca_p = pd_rp->pd_hca;
8419
8420 status = ibt_free_pd(hca_hdl, pd_hdl);
8421 if (status != IBT_SUCCESS) {
8422 return (status);
8423 }
8424 if (DAPLKA_RS_ACCT_CHARGED(pd_rp) > 0) {
8425 DAPLKA_RS_ACCT_DEC(pd_rp, 1);
8426 atomic_dec_32(&hca_p->hca_pd_count);
8427 }
8428 return (status);
8429 }
8430
8431 static ibt_status_t
8432 daplka_ibt_alloc_mw(daplka_mw_resource_t *mw_rp, ibt_hca_hdl_t hca_hdl,
8433 ibt_pd_hdl_t pd_hdl, ibt_mw_flags_t flags, ibt_mw_hdl_t *mw_hdl_p,
8434 ibt_rkey_t *rkey_p)
8435 {
8436 daplka_hca_t *hca_p;
8437 uint32_t max_mws;
8438 boolean_t acct_enabled;
8439 ibt_status_t status;
8440
8441 acct_enabled = daplka_accounting_enabled;
8442 hca_p = mw_rp->mw_hca;
8443 max_mws = daplka_max_mw_percent * hca_p->hca_attr.hca_max_mem_win / 100;
8444
8445 if (acct_enabled) {
8446 if (daplka_max_mw_percent != 0 &&
8447 max_mws <= hca_p->hca_mw_count) {
8448 DERR("ibt_alloc_mw: resource limit exceeded "
8449 "(limit %d, count %d)\n", max_mws,
8450 hca_p->hca_mw_count);
8451 return (IBT_INSUFF_RESOURCE);
8452 }
8453 DAPLKA_RS_ACCT_INC(mw_rp, 1);
8454 atomic_inc_32(&hca_p->hca_mw_count);
8455 }
8456 status = ibt_alloc_mw(hca_hdl, pd_hdl, flags, mw_hdl_p, rkey_p);
8457
8458 if (status != IBT_SUCCESS && acct_enabled) {
8459 DAPLKA_RS_ACCT_DEC(mw_rp, 1);
8460 atomic_dec_32(&hca_p->hca_mw_count);
8461 }
8462 return (status);
8463 }
8464
8465 static ibt_status_t
8466 daplka_ibt_free_mw(daplka_mw_resource_t *mw_rp, ibt_hca_hdl_t hca_hdl,
8467 ibt_mw_hdl_t mw_hdl)
8468 {
8469 daplka_hca_t *hca_p;
8470 ibt_status_t status;
8471
8472 hca_p = mw_rp->mw_hca;
8473
8474 status = ibt_free_mw(hca_hdl, mw_hdl);
8475 if (status != IBT_SUCCESS) {
8476 return (status);
8477 }
8478 if (DAPLKA_RS_ACCT_CHARGED(mw_rp) > 0) {
8479 DAPLKA_RS_ACCT_DEC(mw_rp, 1);
8480 atomic_dec_32(&hca_p->hca_mw_count);
8481 }
8482 return (status);
8483 }
8484
8485 static ibt_status_t
8486 daplka_ibt_register_mr(daplka_mr_resource_t *mr_rp, ibt_hca_hdl_t hca_hdl,
8487 ibt_pd_hdl_t pd_hdl, ibt_mr_attr_t *mr_attr, ibt_mr_hdl_t *mr_hdl_p,
8488 ibt_mr_desc_t *mr_desc_p)
8489 {
8490 daplka_hca_t *hca_p;
8491 uint32_t max_mrs;
8492 boolean_t acct_enabled;
8493 ibt_status_t status;
8494
8495 acct_enabled = daplka_accounting_enabled;
8496 hca_p = mr_rp->mr_hca;
8497 max_mrs = daplka_max_mr_percent * hca_p->hca_attr.hca_max_memr / 100;
8498
8499 if (acct_enabled) {
8500 if (daplka_max_mr_percent != 0 &&
8501 max_mrs <= hca_p->hca_mr_count) {
8502 DERR("ibt_register_mr: resource limit exceeded "
8503 "(limit %d, count %d)\n", max_mrs,
8504 hca_p->hca_mr_count);
8505 return (IBT_INSUFF_RESOURCE);
8506 }
8507 DAPLKA_RS_ACCT_INC(mr_rp, 1);
8508 atomic_inc_32(&hca_p->hca_mr_count);
8509 }
8510 status = ibt_register_mr(hca_hdl, pd_hdl, mr_attr, mr_hdl_p, mr_desc_p);
8511
8512 if (status != IBT_SUCCESS && acct_enabled) {
8513 DAPLKA_RS_ACCT_DEC(mr_rp, 1);
8514 atomic_dec_32(&hca_p->hca_mr_count);
8515 }
8516 return (status);
8517 }
8518
8519 static ibt_status_t
8520 daplka_ibt_register_shared_mr(daplka_mr_resource_t *mr_rp,
8521 ibt_hca_hdl_t hca_hdl, ibt_mr_hdl_t mr_hdl, ibt_pd_hdl_t pd_hdl,
8522 ibt_smr_attr_t *smr_attr_p, ibt_mr_hdl_t *mr_hdl_p,
8523 ibt_mr_desc_t *mr_desc_p)
8524 {
8525 daplka_hca_t *hca_p;
8526 uint32_t max_mrs;
8527 boolean_t acct_enabled;
8528 ibt_status_t status;
8529
8530 acct_enabled = daplka_accounting_enabled;
8531 hca_p = mr_rp->mr_hca;
8532 max_mrs = daplka_max_mr_percent * hca_p->hca_attr.hca_max_memr / 100;
8533
8534 if (acct_enabled) {
8535 if (daplka_max_mr_percent != 0 &&
8536 max_mrs <= hca_p->hca_mr_count) {
8537 DERR("ibt_register_shared_mr: resource limit exceeded "
8538 "(limit %d, count %d)\n", max_mrs,
8539 hca_p->hca_mr_count);
8540 return (IBT_INSUFF_RESOURCE);
8541 }
8542 DAPLKA_RS_ACCT_INC(mr_rp, 1);
8543 atomic_inc_32(&hca_p->hca_mr_count);
8544 }
8545 status = ibt_register_shared_mr(hca_hdl, mr_hdl, pd_hdl,
8546 smr_attr_p, mr_hdl_p, mr_desc_p);
8547
8548 if (status != IBT_SUCCESS && acct_enabled) {
8549 DAPLKA_RS_ACCT_DEC(mr_rp, 1);
8550 atomic_dec_32(&hca_p->hca_mr_count);
8551 }
8552 return (status);
8553 }
8554
8555 static ibt_status_t
8556 daplka_ibt_deregister_mr(daplka_mr_resource_t *mr_rp, ibt_hca_hdl_t hca_hdl,
8557 ibt_mr_hdl_t mr_hdl)
8558 {
8559 daplka_hca_t *hca_p;
8560 ibt_status_t status;
8561
8562 hca_p = mr_rp->mr_hca;
8563
8564 status = ibt_deregister_mr(hca_hdl, mr_hdl);
8565 if (status != IBT_SUCCESS) {
8566 return (status);
8567 }
8568 if (DAPLKA_RS_ACCT_CHARGED(mr_rp) > 0) {
8569 DAPLKA_RS_ACCT_DEC(mr_rp, 1);
8570 atomic_dec_32(&hca_p->hca_mr_count);
8571 }
8572 return (status);
8573 }
8574
8575 static ibt_status_t
8576 daplka_ibt_alloc_srq(daplka_srq_resource_t *srq_rp, ibt_hca_hdl_t hca_hdl,
8577 ibt_srq_flags_t flags, ibt_pd_hdl_t pd, ibt_srq_sizes_t *reqsz,
8578 ibt_srq_hdl_t *srq_hdl_p, ibt_srq_sizes_t *realsz)
8579 {
8580 daplka_hca_t *hca_p;
8581 uint32_t max_srqs;
8582 boolean_t acct_enabled;
8583 ibt_status_t status;
8584
8585 acct_enabled = daplka_accounting_enabled;
8586 hca_p = srq_rp->srq_hca;
8587 max_srqs = daplka_max_srq_percent * hca_p->hca_attr.hca_max_srqs / 100;
8588
8589 if (acct_enabled) {
8590 if (daplka_max_srq_percent != 0 &&
8591 max_srqs <= hca_p->hca_srq_count) {
8592 DERR("ibt_alloc_srq: resource limit exceeded "
8593 "(limit %d, count %d)\n", max_srqs,
8594 hca_p->hca_srq_count);
8595 return (IBT_INSUFF_RESOURCE);
8596 }
8597 DAPLKA_RS_ACCT_INC(srq_rp, 1);
8598 atomic_inc_32(&hca_p->hca_srq_count);
8599 }
8600 status = ibt_alloc_srq(hca_hdl, flags, pd, reqsz, srq_hdl_p, realsz);
8601
8602 if (status != IBT_SUCCESS && acct_enabled) {
8603 DAPLKA_RS_ACCT_DEC(srq_rp, 1);
8604 atomic_dec_32(&hca_p->hca_srq_count);
8605 }
8606 return (status);
8607 }
8608
8609 static ibt_status_t
8610 daplka_ibt_free_srq(daplka_srq_resource_t *srq_rp, ibt_srq_hdl_t srq_hdl)
8611 {
8612 daplka_hca_t *hca_p;
8613 ibt_status_t status;
8614
8615 hca_p = srq_rp->srq_hca;
8616
8617 D3("ibt_free_srq: %p %p\n", srq_rp, srq_hdl);
8618
8619 status = ibt_free_srq(srq_hdl);
8620 if (status != IBT_SUCCESS) {
8621 return (status);
8622 }
8623 if (DAPLKA_RS_ACCT_CHARGED(srq_rp) > 0) {
8624 DAPLKA_RS_ACCT_DEC(srq_rp, 1);
8625 atomic_dec_32(&hca_p->hca_srq_count);
8626 }
8627 return (status);
8628 }
8629
8630
8631 static int
8632 daplka_common_ioctl(int cmd, minor_t rnum, intptr_t arg, int mode,
8633 cred_t *cred, int *rvalp)
8634 {
8635 int error;
8636
8637 switch (cmd) {
8638 case DAPL_IA_CREATE:
8639 error = daplka_ia_create(rnum, arg, mode, cred, rvalp);
8640 break;
8641
8642 /* can potentially add other commands here */
8643
8644 default:
8645 DERR("daplka_common_ioctl: cmd not supported\n");
8646 error = DDI_FAILURE;
8647 }
8648 return (error);
8649 }
8650
8651 static int
8652 daplka_evd_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8653 cred_t *cred, int *rvalp)
8654 {
8655 int error;
8656
8657 switch (cmd) {
8658 case DAPL_EVD_CREATE:
8659 error = daplka_evd_create(rp, arg, mode, cred, rvalp);
8660 break;
8661
8662 case DAPL_CQ_RESIZE:
8663 error = daplka_cq_resize(rp, arg, mode, cred, rvalp);
8664 break;
8665
8666 case DAPL_EVENT_POLL:
8667 error = daplka_event_poll(rp, arg, mode, cred, rvalp);
8668 break;
8669
8670 case DAPL_EVENT_WAKEUP:
8671 error = daplka_event_wakeup(rp, arg, mode, cred, rvalp);
8672 break;
8673
8674 case DAPL_EVD_MODIFY_CNO:
8675 error = daplka_evd_modify_cno(rp, arg, mode, cred, rvalp);
8676 break;
8677
8678 case DAPL_EVD_FREE:
8679 error = daplka_evd_free(rp, arg, mode, cred, rvalp);
8680 break;
8681
8682 default:
8683 DERR("daplka_evd_ioctl: cmd not supported\n");
8684 error = DDI_FAILURE;
8685 }
8686 return (error);
8687 }
8688
8689 static int
8690 daplka_ep_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8691 cred_t *cred, int *rvalp)
8692 {
8693 int error;
8694
8695 switch (cmd) {
8696 case DAPL_EP_MODIFY:
8697 error = daplka_ep_modify(rp, arg, mode, cred, rvalp);
8698 break;
8699
8700 case DAPL_EP_FREE:
8701 error = daplka_ep_free(rp, arg, mode, cred, rvalp);
8702 break;
8703
8704 case DAPL_EP_CONNECT:
8705 error = daplka_ep_connect(rp, arg, mode, cred, rvalp);
8706 break;
8707
8708 case DAPL_EP_DISCONNECT:
8709 error = daplka_ep_disconnect(rp, arg, mode, cred, rvalp);
8710 break;
8711
8712 case DAPL_EP_REINIT:
8713 error = daplka_ep_reinit(rp, arg, mode, cred, rvalp);
8714 break;
8715
8716 case DAPL_EP_CREATE:
8717 error = daplka_ep_create(rp, arg, mode, cred, rvalp);
8718 break;
8719
8720 default:
8721 DERR("daplka_ep_ioctl: cmd not supported\n");
8722 error = DDI_FAILURE;
8723 }
8724 return (error);
8725 }
8726
8727 static int
8728 daplka_mr_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8729 cred_t *cred, int *rvalp)
8730 {
8731 int error;
8732
8733 switch (cmd) {
8734 case DAPL_MR_REGISTER:
8735 error = daplka_mr_register(rp, arg, mode, cred, rvalp);
8736 break;
8737
8738 case DAPL_MR_REGISTER_LMR:
8739 error = daplka_mr_register_lmr(rp, arg, mode, cred, rvalp);
8740 break;
8741
8742 case DAPL_MR_REGISTER_SHARED:
8743 error = daplka_mr_register_shared(rp, arg, mode, cred, rvalp);
8744 break;
8745
8746 case DAPL_MR_DEREGISTER:
8747 error = daplka_mr_deregister(rp, arg, mode, cred, rvalp);
8748 break;
8749
8750 case DAPL_MR_SYNC:
8751 error = daplka_mr_sync(rp, arg, mode, cred, rvalp);
8752 break;
8753
8754 default:
8755 DERR("daplka_mr_ioctl: cmd not supported\n");
8756 error = DDI_FAILURE;
8757 }
8758 return (error);
8759 }
8760
8761 static int
8762 daplka_mw_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8763 cred_t *cred, int *rvalp)
8764 {
8765 int error;
8766
8767 switch (cmd) {
8768 case DAPL_MW_ALLOC:
8769 error = daplka_mw_alloc(rp, arg, mode, cred, rvalp);
8770 break;
8771
8772 case DAPL_MW_FREE:
8773 error = daplka_mw_free(rp, arg, mode, cred, rvalp);
8774 break;
8775
8776 default:
8777 DERR("daplka_mw_ioctl: cmd not supported\n");
8778 error = DDI_FAILURE;
8779 }
8780 return (error);
8781 }
8782
8783 static int
8784 daplka_cno_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8785 cred_t *cred, int *rvalp)
8786 {
8787 int error;
8788
8789 switch (cmd) {
8790 case DAPL_CNO_ALLOC:
8791 error = daplka_cno_alloc(rp, arg, mode, cred, rvalp);
8792 break;
8793
8794 case DAPL_CNO_FREE:
8795 error = daplka_cno_free(rp, arg, mode, cred, rvalp);
8796 break;
8797
8798 case DAPL_CNO_WAIT:
8799 error = daplka_cno_wait(rp, arg, mode, cred, rvalp);
8800 break;
8801
8802 default:
8803 DERR("daplka_cno_ioctl: cmd not supported\n");
8804 error = DDI_FAILURE;
8805 }
8806 return (error);
8807 }
8808
8809 static int
8810 daplka_pd_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8811 cred_t *cred, int *rvalp)
8812 {
8813 int error;
8814
8815 switch (cmd) {
8816 case DAPL_PD_ALLOC:
8817 error = daplka_pd_alloc(rp, arg, mode, cred, rvalp);
8818 break;
8819
8820 case DAPL_PD_FREE:
8821 error = daplka_pd_free(rp, arg, mode, cred, rvalp);
8822 break;
8823
8824 default:
8825 DERR("daplka_pd_ioctl: cmd not supported\n");
8826 error = DDI_FAILURE;
8827 }
8828 return (error);
8829 }
8830
8831 static int
8832 daplka_sp_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8833 cred_t *cred, int *rvalp)
8834 {
8835 int error;
8836
8837 switch (cmd) {
8838 case DAPL_SERVICE_REGISTER:
8839 error = daplka_service_register(rp, arg, mode, cred, rvalp);
8840 break;
8841
8842 case DAPL_SERVICE_DEREGISTER:
8843 error = daplka_service_deregister(rp, arg, mode, cred, rvalp);
8844 break;
8845
8846 default:
8847 DERR("daplka_sp_ioctl: cmd not supported\n");
8848 error = DDI_FAILURE;
8849 }
8850 return (error);
8851 }
8852
8853 static int
8854 daplka_srq_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8855 cred_t *cred, int *rvalp)
8856 {
8857 int error;
8858
8859 switch (cmd) {
8860 case DAPL_SRQ_CREATE:
8861 error = daplka_srq_create(rp, arg, mode, cred, rvalp);
8862 break;
8863
8864 case DAPL_SRQ_RESIZE:
8865 error = daplka_srq_resize(rp, arg, mode, cred, rvalp);
8866 break;
8867
8868 case DAPL_SRQ_FREE:
8869 error = daplka_srq_free(rp, arg, mode, cred, rvalp);
8870 break;
8871
8872 default:
8873 DERR("daplka_srq_ioctl: cmd(%d) not supported\n", cmd);
8874 error = DDI_FAILURE;
8875 break;
8876 }
8877 return (error);
8878 }
8879
8880 static int
8881 daplka_misc_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
8882 cred_t *cred, int *rvalp)
8883 {
8884 int error;
8885
8886 switch (cmd) {
8887 case DAPL_CR_ACCEPT:
8888 error = daplka_cr_accept(rp, arg, mode, cred, rvalp);
8889 break;
8890
8891 case DAPL_CR_REJECT:
8892 error = daplka_cr_reject(rp, arg, mode, cred, rvalp);
8893 break;
8894
8895 case DAPL_IA_QUERY:
8896 error = daplka_ia_query(rp, arg, mode, cred, rvalp);
8897 break;
8898
8899 case DAPL_CR_HANDOFF:
8900 error = daplka_cr_handoff(rp, arg, mode, cred, rvalp);
8901 break;
8902
8903 default:
8904 DERR("daplka_misc_ioctl: cmd not supported\n");
8905 error = DDI_FAILURE;
8906 }
8907 return (error);
8908 }
8909
8910 /*ARGSUSED*/
8911 static int
8912 daplka_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
8913 int *rvalp)
8914 {
8915 daplka_ia_resource_t *ia_rp;
8916 minor_t rnum;
8917 int error = 0;
8918
8919 rnum = getminor(dev);
8920 ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(rnum);
8921 if (ia_rp == NULL) {
8922 DERR("ioctl: resource not found, rnum %d\n", rnum);
8923 return (ENXIO);
8924 }
8925
8926 D4("ioctl: rnum = %d, cmd = 0x%x\n", rnum, cmd);
8927 if (DAPLKA_RS_RESERVED(ia_rp)) {
8928 error = daplka_common_ioctl(cmd, rnum, arg, mode, cred, rvalp);
8929 return (error);
8930 }
8931 if (DAPLKA_RS_TYPE(ia_rp) != DAPL_TYPE_IA) {
8932 DERR("ioctl: invalid type %d\n", DAPLKA_RS_TYPE(ia_rp));
8933 error = EINVAL;
8934 goto cleanup;
8935 }
8936 if (ia_rp->ia_pid != ddi_get_pid()) {
8937 DERR("ioctl: ia_pid %d != pid %d\n",
8938 ia_rp->ia_pid, ddi_get_pid());
8939 error = EINVAL;
8940 goto cleanup;
8941 }
8942
8943 switch (cmd & DAPL_TYPE_MASK) {
8944 case DAPL_TYPE_EVD:
8945 error = daplka_evd_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8946 break;
8947
8948 case DAPL_TYPE_EP:
8949 error = daplka_ep_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8950 break;
8951
8952 case DAPL_TYPE_MR:
8953 error = daplka_mr_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8954 break;
8955
8956 case DAPL_TYPE_MW:
8957 error = daplka_mw_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8958 break;
8959
8960 case DAPL_TYPE_PD:
8961 error = daplka_pd_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8962 break;
8963
8964 case DAPL_TYPE_SP:
8965 error = daplka_sp_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8966 break;
8967
8968 case DAPL_TYPE_CNO:
8969 error = daplka_cno_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8970 break;
8971
8972 case DAPL_TYPE_MISC:
8973 error = daplka_misc_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8974 break;
8975
8976 case DAPL_TYPE_SRQ:
8977 error = daplka_srq_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
8978 break;
8979
8980 default:
8981 DERR("ioctl: invalid dapl type = %d\n", DAPLKA_RS_TYPE(ia_rp));
8982 error = DDI_FAILURE;
8983 }
8984
8985 cleanup:;
8986 DAPLKA_RS_UNREF(ia_rp);
8987 return (error);
8988 }
8989
8990 /* ARGSUSED */
8991 static int
8992 daplka_open(dev_t *devp, int flag, int otyp, struct cred *cred)
8993 {
8994 minor_t rnum;
8995
8996 /*
8997 * Char only
8998 */
8999 if (otyp != OTYP_CHR) {
9000 return (EINVAL);
9001 }
9002
9003 /*
9004 * Only zero can be opened, clones are used for resources.
9005 */
9006 if (getminor(*devp) != DAPLKA_DRIVER_MINOR) {
9007 DERR("daplka_open: bad minor %d\n", getminor(*devp));
9008 return (ENODEV);
9009 }
9010
9011 /*
9012 * - allocate new minor number
9013 * - update devp argument to new device
9014 */
9015 if (daplka_resource_reserve(&rnum) == 0) {
9016 *devp = makedevice(getmajor(*devp), rnum);
9017 } else {
9018 return (ENOMEM);
9019 }
9020
9021 return (DDI_SUCCESS);
9022 }
9023
9024 /* ARGSUSED */
9025 static int
9026 daplka_close(dev_t dev, int flag, int otyp, struct cred *cred)
9027 {
9028 daplka_ia_resource_t *ia_rp;
9029 minor_t rnum = getminor(dev);
9030
9031 /*
9032 * Char only
9033 */
9034 if (otyp != OTYP_CHR) {
9035 return (EINVAL);
9036 }
9037 D2("daplka_close: closing rnum = %d\n", rnum);
9038 atomic_inc_32(&daplka_pending_close);
9039
9040 /*
9041 * remove from resource table.
9042 */
9043 ia_rp = (daplka_ia_resource_t *)daplka_resource_remove(rnum);
9044
9045 /*
9046 * remove the initial reference
9047 */
9048 if (ia_rp != NULL) {
9049 DAPLKA_RS_UNREF(ia_rp);
9050 }
9051 atomic_dec_32(&daplka_pending_close);
9052 return (DDI_SUCCESS);
9053 }
9054
9055
9056 /*
9057 * Resource management routines
9058 *
9059 * We start with no resource array. Each time we run out of slots, we
9060 * reallocate a new larger array and copy the pointer to the new array and
9061 * a new resource blk is allocated and added to the hash table.
9062 *
9063 * The resource control block contains:
9064 * root - array of pointer of resource blks
9065 * sz - current size of array.
9066 * len - last valid entry in array.
9067 *
9068 * A search operation based on a resource number is as follows:
9069 * index = rnum / RESOURCE_BLKSZ;
9070 * ASSERT(index < resource_block.len);
9071 * ASSERT(index < resource_block.sz);
9072 * offset = rnum % RESOURCE_BLKSZ;
9073 * ASSERT(offset >= resource_block.root[index]->base);
9074 * ASSERT(offset < resource_block.root[index]->base + RESOURCE_BLKSZ);
9075 * return resource_block.root[index]->blks[offset];
9076 *
9077 * A resource blk is freed when its used count reaches zero.
9078 */
9079
9080 /*
9081 * initializes the global resource table
9082 */
9083 static void
9084 daplka_resource_init(void)
9085 {
9086 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(daplka_resource))
9087 rw_init(&daplka_resource.daplka_rct_lock, NULL, RW_DRIVER, NULL);
9088 daplka_resource.daplka_rc_len = 0;
9089 daplka_resource.daplka_rc_sz = 0;
9090 daplka_resource.daplka_rc_cnt = 0;
9091 daplka_resource.daplka_rc_flag = 0;
9092 daplka_resource.daplka_rc_root = NULL;
9093 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(daplka_resource))
9094 }
9095
9096 /*
9097 * destroys the global resource table
9098 */
9099 static void
9100 daplka_resource_fini(void)
9101 {
9102 int i;
9103
9104 rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9105 for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9106 daplka_resource_blk_t *blk;
9107 int j;
9108
9109 blk = daplka_resource.daplka_rc_root[i];
9110 if (blk == NULL) {
9111 continue;
9112 }
9113 for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9114 if (blk->daplka_rcblk_blks[j] != NULL) {
9115 DERR("resource_fini: non-null slot %d, %p\n",
9116 j, blk->daplka_rcblk_blks[j]);
9117 }
9118 }
9119 kmem_free(blk, sizeof (*blk));
9120 daplka_resource.daplka_rc_root[i] = NULL;
9121 }
9122 if (daplka_resource.daplka_rc_root != NULL) {
9123 uint_t sz;
9124
9125 sz = daplka_resource.daplka_rc_sz *
9126 sizeof (daplka_resource_blk_t *);
9127 kmem_free(daplka_resource.daplka_rc_root, (uint_t)sz);
9128 daplka_resource.daplka_rc_root = NULL;
9129 daplka_resource.daplka_rc_len = 0;
9130 daplka_resource.daplka_rc_sz = 0;
9131 }
9132 rw_exit(&daplka_resource.daplka_rct_lock);
9133 rw_destroy(&daplka_resource.daplka_rct_lock);
9134 }
9135
9136 /*
9137 * reserves a slot in the global resource table.
9138 * this is called by the open() syscall. it is needed because
9139 * at open() time, we do not have sufficient information to
9140 * create an IA resource. the library needs to subsequently
9141 * call daplka_ia_create to insert an IA resource into this
9142 * reserved slot.
9143 */
9144 static int
9145 daplka_resource_reserve(minor_t *rnum)
9146 {
9147 int i, j, empty = -1;
9148 daplka_resource_blk_t *blk;
9149
9150 rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9151 /*
9152 * Try to find an empty slot
9153 */
9154 for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9155 blk = daplka_resource.daplka_rc_root[i];
9156 if (blk != NULL && blk->daplka_rcblk_avail > 0) {
9157
9158 D3("resource_alloc: available blks %d\n",
9159 blk->daplka_rcblk_avail);
9160
9161 /*
9162 * found an empty slot in this blk
9163 */
9164 for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9165 if (blk->daplka_rcblk_blks[j] == NULL) {
9166 *rnum = (minor_t)
9167 (j + (i * DAPLKA_RC_BLKSZ));
9168 blk->daplka_rcblk_blks[j] =
9169 (daplka_resource_t *)
9170 DAPLKA_RC_RESERVED;
9171 blk->daplka_rcblk_avail--;
9172 daplka_resource.daplka_rc_cnt++;
9173 rw_exit(&daplka_resource.
9174 daplka_rct_lock);
9175 return (0);
9176 }
9177 }
9178 } else if (blk == NULL && empty < 0) {
9179 /*
9180 * remember first empty slot
9181 */
9182 empty = i;
9183 }
9184 }
9185
9186 /*
9187 * Couldn't find anything, allocate a new blk
9188 * Do we need to reallocate the root array
9189 */
9190 if (empty < 0) {
9191 if (daplka_resource.daplka_rc_len ==
9192 daplka_resource.daplka_rc_sz) {
9193 /*
9194 * Allocate new array and copy current stuff into it
9195 */
9196 daplka_resource_blk_t **p;
9197 uint_t newsz = (uint_t)daplka_resource.daplka_rc_sz +
9198 DAPLKA_RC_BLKSZ;
9199
9200 D3("resource_alloc: increasing no. of buckets to %d\n",
9201 newsz);
9202
9203 p = kmem_zalloc(newsz * sizeof (*p), daplka_km_flags);
9204
9205 if (daplka_resource.daplka_rc_root) {
9206 uint_t oldsz;
9207
9208 oldsz = (uint_t)(daplka_resource.daplka_rc_sz *
9209 (int)sizeof (*p));
9210
9211 /*
9212 * Copy old data into new space and
9213 * free old stuff
9214 */
9215 bcopy(daplka_resource.daplka_rc_root, p, oldsz);
9216 kmem_free(daplka_resource.daplka_rc_root,
9217 oldsz);
9218 }
9219
9220 daplka_resource.daplka_rc_root = p;
9221 daplka_resource.daplka_rc_sz = (int)newsz;
9222 }
9223
9224 empty = daplka_resource.daplka_rc_len;
9225 daplka_resource.daplka_rc_len++;
9226
9227 D3("resource_alloc: daplka_rc_len %d\n",
9228 daplka_resource.daplka_rc_len);
9229 }
9230
9231 /*
9232 * Allocate a new blk
9233 */
9234 blk = kmem_zalloc(sizeof (*blk), daplka_km_flags);
9235 ASSERT(daplka_resource.daplka_rc_root[empty] == NULL);
9236 daplka_resource.daplka_rc_root[empty] = blk;
9237 blk->daplka_rcblk_avail = DAPLKA_RC_BLKSZ - 1;
9238
9239 /*
9240 * Allocate slot
9241 */
9242 *rnum = (minor_t)(empty * DAPLKA_RC_BLKSZ);
9243 blk->daplka_rcblk_blks[0] = (daplka_resource_t *)DAPLKA_RC_RESERVED;
9244 daplka_resource.daplka_rc_cnt++;
9245 rw_exit(&daplka_resource.daplka_rct_lock);
9246
9247 return (0);
9248 }
9249
9250 /*
9251 * removes resource from global resource table
9252 */
9253 static daplka_resource_t *
9254 daplka_resource_remove(minor_t rnum)
9255 {
9256 int i, j;
9257 daplka_resource_blk_t *blk;
9258 daplka_resource_t *p;
9259
9260 i = (int)(rnum / DAPLKA_RC_BLKSZ);
9261 j = (int)(rnum % DAPLKA_RC_BLKSZ);
9262
9263 rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9264 if (i >= daplka_resource.daplka_rc_len) {
9265 rw_exit(&daplka_resource.daplka_rct_lock);
9266 DERR("resource_remove: invalid rnum %d\n", rnum);
9267 return (NULL);
9268 }
9269
9270 ASSERT(daplka_resource.daplka_rc_root);
9271 ASSERT(i < daplka_resource.daplka_rc_len);
9272 ASSERT(i < daplka_resource.daplka_rc_sz);
9273 blk = daplka_resource.daplka_rc_root[i];
9274 if (blk == NULL) {
9275 rw_exit(&daplka_resource.daplka_rct_lock);
9276 DERR("resource_remove: invalid rnum %d\n", rnum);
9277 return (NULL);
9278 }
9279
9280 if (blk->daplka_rcblk_blks[j] == NULL) {
9281 rw_exit(&daplka_resource.daplka_rct_lock);
9282 DERR("resource_remove: blk->daplka_rcblk_blks[j] == NULL\n");
9283 return (NULL);
9284 }
9285 p = blk->daplka_rcblk_blks[j];
9286 blk->daplka_rcblk_blks[j] = NULL;
9287 blk->daplka_rcblk_avail++;
9288 if (blk->daplka_rcblk_avail == DAPLKA_RC_BLKSZ) {
9289 /*
9290 * free this blk
9291 */
9292 kmem_free(blk, sizeof (*blk));
9293 daplka_resource.daplka_rc_root[i] = NULL;
9294 }
9295 daplka_resource.daplka_rc_cnt--;
9296 rw_exit(&daplka_resource.daplka_rct_lock);
9297
9298 if ((intptr_t)p == DAPLKA_RC_RESERVED) {
9299 return (NULL);
9300 } else {
9301 return (p);
9302 }
9303 }
9304
9305 /*
9306 * inserts resource into the slot designated by rnum
9307 */
9308 static int
9309 daplka_resource_insert(minor_t rnum, daplka_resource_t *rp)
9310 {
9311 int i, j, error = -1;
9312 daplka_resource_blk_t *blk;
9313
9314 /*
9315 * Find resource and lock it in WRITER mode
9316 * search for available resource slot
9317 */
9318
9319 i = (int)(rnum / DAPLKA_RC_BLKSZ);
9320 j = (int)(rnum % DAPLKA_RC_BLKSZ);
9321
9322 rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9323 if (i >= daplka_resource.daplka_rc_len) {
9324 rw_exit(&daplka_resource.daplka_rct_lock);
9325 DERR("resource_insert: resource %d not found\n", rnum);
9326 return (-1);
9327 }
9328
9329 blk = daplka_resource.daplka_rc_root[i];
9330 if (blk != NULL) {
9331 ASSERT(i < daplka_resource.daplka_rc_len);
9332 ASSERT(i < daplka_resource.daplka_rc_sz);
9333
9334 if ((intptr_t)blk->daplka_rcblk_blks[j] == DAPLKA_RC_RESERVED) {
9335 blk->daplka_rcblk_blks[j] = rp;
9336 error = 0;
9337 } else {
9338 DERR("resource_insert: %d not reserved, blk = %p\n",
9339 rnum, blk->daplka_rcblk_blks[j]);
9340 }
9341 } else {
9342 DERR("resource_insert: resource %d not found\n", rnum);
9343 }
9344 rw_exit(&daplka_resource.daplka_rct_lock);
9345 return (error);
9346 }
9347
9348 /*
9349 * finds resource using minor device number
9350 */
9351 static daplka_resource_t *
9352 daplka_resource_lookup(minor_t rnum)
9353 {
9354 int i, j;
9355 daplka_resource_blk_t *blk;
9356 daplka_resource_t *rp;
9357
9358 /*
9359 * Find resource and lock it in READER mode
9360 * search for available resource slot
9361 */
9362
9363 i = (int)(rnum / DAPLKA_RC_BLKSZ);
9364 j = (int)(rnum % DAPLKA_RC_BLKSZ);
9365
9366 rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
9367 if (i >= daplka_resource.daplka_rc_len) {
9368 rw_exit(&daplka_resource.daplka_rct_lock);
9369 DERR("resource_lookup: resource %d not found\n", rnum);
9370 return (NULL);
9371 }
9372
9373 blk = daplka_resource.daplka_rc_root[i];
9374 if (blk != NULL) {
9375 ASSERT(i < daplka_resource.daplka_rc_len);
9376 ASSERT(i < daplka_resource.daplka_rc_sz);
9377
9378 rp = blk->daplka_rcblk_blks[j];
9379 if (rp == NULL || (intptr_t)rp == DAPLKA_RC_RESERVED) {
9380 D3("resource_lookup: %d not found, blk = %p\n",
9381 rnum, blk->daplka_rcblk_blks[j]);
9382 } else {
9383 DAPLKA_RS_REF((daplka_ia_resource_t *)rp);
9384 }
9385 } else {
9386 DERR("resource_lookup: resource %d not found\n", rnum);
9387 rp = NULL;
9388 }
9389 rw_exit(&daplka_resource.daplka_rct_lock);
9390 return (rp);
9391 }
9392
9393 /*
9394 * generic hash table implementation
9395 */
9396
9397 /*
9398 * daplka_hash_create:
9399 * initializes a hash table with the specified parameters
9400 *
9401 * input:
9402 * htblp pointer to hash table
9403 *
9404 * nbuckets number of buckets (must be power of 2)
9405 *
9406 * free_func this function is called on each hash
9407 * table element when daplka_hash_destroy
9408 * is called
9409 *
9410 * lookup_func if daplka_hash_lookup is able to find
9411 * the desired object, this function is
9412 * applied on the object before
9413 * daplka_hash_lookup returns
9414 * output:
9415 * none
9416 *
9417 * return value(s):
9418 * EINVAL nbuckets is not a power of 2
9419 * ENOMEM cannot allocate buckets
9420 * 0 success
9421 */
9422 static int
9423 daplka_hash_create(daplka_hash_table_t *htblp, uint_t nbuckets,
9424 void (*free_func)(void *), void (*lookup_func)(void *))
9425 {
9426 int i;
9427
9428 if ((nbuckets & ~(nbuckets - 1)) != nbuckets) {
9429 DERR("hash_create: nbuckets not power of 2\n");
9430 return (EINVAL);
9431 }
9432 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*htblp))
9433
9434 htblp->ht_buckets =
9435 kmem_zalloc(sizeof (daplka_hash_bucket_t) * nbuckets,
9436 daplka_km_flags);
9437 if (htblp->ht_buckets == NULL) {
9438 DERR("hash_create: cannot allocate buckets\n");
9439 return (ENOMEM);
9440 }
9441 for (i = 0; i < nbuckets; i++) {
9442 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(htblp->ht_buckets[i]))
9443 htblp->ht_buckets[i].hb_count = 0;
9444 htblp->ht_buckets[i].hb_entries = NULL;
9445 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(htblp->ht_buckets[i]))
9446 }
9447 rw_init(&htblp->ht_table_lock, NULL, RW_DRIVER, NULL);
9448 mutex_init(&htblp->ht_key_lock, NULL, MUTEX_DRIVER, NULL);
9449
9450 htblp->ht_count = 0;
9451 htblp->ht_next_hkey = (uint64_t)gethrtime();
9452 htblp->ht_nbuckets = nbuckets;
9453 htblp->ht_free_func = free_func;
9454 htblp->ht_lookup_func = lookup_func;
9455 htblp->ht_initialized = B_TRUE;
9456 D3("hash_create: done, buckets = %d\n", nbuckets);
9457 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*htblp))
9458 return (0);
9459 }
9460
9461 /*
9462 * daplka_hash_insert:
9463 * inserts an object into a hash table
9464 *
9465 * input:
9466 * htblp pointer to hash table
9467 *
9468 * hkeyp pointer to hash key.
9469 * *hkeyp being non-zero means that the caller
9470 * has generated its own hkey. if *hkeyp is zero,
9471 * this function will generate an hkey for the
9472 * caller. it is recommended that the caller
9473 * leave the hkey generation to this function
9474 * because the hkey is more likely to be evenly
9475 * distributed.
9476 *
9477 * objp pointer to object to be inserted into
9478 * hash table
9479 *
9480 * output:
9481 * hkeyp the generated hkey is returned via this pointer
9482 *
9483 * return value(s):
9484 * EINVAL invalid parameter
9485 * ENOMEM cannot allocate hash entry
9486 * 0 successful
9487 */
9488 static int
9489 daplka_hash_insert(daplka_hash_table_t *htblp, uint64_t *hkeyp, void *objp)
9490 {
9491 daplka_hash_entry_t *hep, *curr_hep;
9492 daplka_hash_bucket_t *hbp;
9493 uint32_t bucket;
9494 uint64_t hkey;
9495
9496 if (hkeyp == NULL) {
9497 DERR("hash_insert: hkeyp == NULL\n");
9498 return (EINVAL);
9499 }
9500 hep = kmem_zalloc(sizeof (*hep), daplka_km_flags);
9501 if (hep == NULL) {
9502 DERR("hash_insert: cannot alloc hash_entry\n");
9503 return (ENOMEM);
9504 }
9505 if (*hkeyp == 0) {
9506 /* generate a new key */
9507 mutex_enter(&htblp->ht_key_lock);
9508 hkey = ++htblp->ht_next_hkey;
9509 if (hkey == 0) {
9510 hkey = htblp->ht_next_hkey = (uint64_t)gethrtime();
9511 }
9512 mutex_exit(&htblp->ht_key_lock);
9513 } else {
9514 /* use user generated key */
9515 hkey = *hkeyp;
9516 }
9517
9518 /* only works if ht_nbuckets is a power of 2 */
9519 bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9520 ASSERT(objp != NULL);
9521 ASSERT(bucket < htblp->ht_nbuckets);
9522
9523 rw_enter(&htblp->ht_table_lock, RW_WRITER);
9524 hep->he_hkey = hkey;
9525 hep->he_objp = objp;
9526
9527 /* look for duplicate entries */
9528 hbp = &htblp->ht_buckets[bucket];
9529 curr_hep = hbp->hb_entries;
9530 while (curr_hep != NULL) {
9531 if (curr_hep->he_hkey == hep->he_hkey) {
9532 break;
9533 }
9534 curr_hep = curr_hep->he_next;
9535 }
9536 if (curr_hep != NULL) {
9537 DERR("hash_insert: found duplicate hash entry: "
9538 "bucket %d, hkey 0x%016llx\n",
9539 bucket, (longlong_t)hep->he_hkey);
9540 kmem_free(hep, sizeof (*hep));
9541 rw_exit(&htblp->ht_table_lock);
9542 return (EINVAL);
9543 }
9544 hep->he_next = hbp->hb_entries;
9545 hbp->hb_entries = hep;
9546 hbp->hb_count++;
9547 htblp->ht_count++;
9548 rw_exit(&htblp->ht_table_lock);
9549
9550 if (*hkeyp == 0) {
9551 *hkeyp = hkey;
9552 ASSERT(*hkeyp != 0);
9553 }
9554 D3("hash_insert: htblp 0x%p, hkey = 0x%016llx, bucket = %d\n",
9555 htblp, (longlong_t)*hkeyp, bucket);
9556 return (0);
9557 }
9558
9559 /*
9560 * daplka_hash_remove:
9561 * removes object identified by hkey from hash table
9562 *
9563 * input:
9564 * htblp pointer to hash table
9565 *
9566 * hkey hkey that identifies the object to be removed
9567 *
9568 * output:
9569 * objpp pointer to pointer to object.
9570 * if remove is successful, the removed object
9571 * will be returned via *objpp.
9572 *
9573 * return value(s):
9574 * EINVAL cannot find hash entry
9575 * 0 successful
9576 */
9577 static int
9578 daplka_hash_remove(daplka_hash_table_t *htblp, uint64_t hkey, void **objpp)
9579 {
9580 daplka_hash_entry_t *free_hep, **curr_hepp;
9581 daplka_hash_bucket_t *hbp;
9582 uint32_t bucket;
9583
9584 bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9585
9586 rw_enter(&htblp->ht_table_lock, RW_WRITER);
9587 hbp = &htblp->ht_buckets[bucket];
9588
9589 curr_hepp = &hbp->hb_entries;
9590 while (*curr_hepp != NULL) {
9591 if ((*curr_hepp)->he_hkey == hkey) {
9592 break;
9593 }
9594 curr_hepp = &(*curr_hepp)->he_next;
9595 }
9596 if (*curr_hepp == NULL) {
9597 DERR("hash_remove: cannot find hash entry: "
9598 "bucket %d, hkey 0x%016llx\n", bucket, (longlong_t)hkey);
9599 rw_exit(&htblp->ht_table_lock);
9600 return (EINVAL);
9601 } else {
9602 if (objpp != NULL) {
9603 *objpp = (*curr_hepp)->he_objp;
9604 }
9605 free_hep = *curr_hepp;
9606 *curr_hepp = (*curr_hepp)->he_next;
9607 kmem_free(free_hep, sizeof (*free_hep));
9608 }
9609 hbp->hb_count--;
9610 htblp->ht_count--;
9611 D3("hash_remove: removed entry, hkey 0x%016llx, bucket %d, "
9612 "hb_count %d, hb_count %d\n",
9613 (longlong_t)hkey, bucket, hbp->hb_count, htblp->ht_count);
9614 rw_exit(&htblp->ht_table_lock);
9615 return (0);
9616 }
9617
9618 /*
9619 * daplka_hash_walk:
9620 * walks through the entire hash table. applying func on each of
9621 * the inserted objects. stops walking if func returns non-zero.
9622 *
9623 * input:
9624 * htblp pointer to hash table
9625 *
9626 * func function to be applied on each object
9627 *
9628 * farg second argument to func
9629 *
9630 * lockmode can be RW_WRITER or RW_READER. this
9631 * allows the caller to choose what type
9632 * of lock to acquire before walking the
9633 * table.
9634 *
9635 * output:
9636 * none
9637 *
9638 * return value(s):
9639 * none
9640 */
9641 static void
9642 daplka_hash_walk(daplka_hash_table_t *htblp, int (*func)(void *, void *),
9643 void *farg, krw_t lockmode)
9644 {
9645 daplka_hash_entry_t *curr_hep;
9646 daplka_hash_bucket_t *hbp;
9647 uint32_t bucket, retval = 0;
9648
9649 ASSERT(lockmode == RW_WRITER || lockmode == RW_READER);
9650
9651 /* needed for warlock */
9652 if (lockmode == RW_WRITER) {
9653 rw_enter(&htblp->ht_table_lock, RW_WRITER);
9654 } else {
9655 rw_enter(&htblp->ht_table_lock, RW_READER);
9656 }
9657 for (bucket = 0; bucket < htblp->ht_nbuckets && retval == 0; bucket++) {
9658 hbp = &htblp->ht_buckets[bucket];
9659 curr_hep = hbp->hb_entries;
9660 while (curr_hep != NULL) {
9661 retval = (*func)(curr_hep->he_objp, farg);
9662 if (retval != 0) {
9663 break;
9664 }
9665 curr_hep = curr_hep->he_next;
9666 }
9667 }
9668 rw_exit(&htblp->ht_table_lock);
9669 }
9670
9671 /*
9672 * daplka_hash_lookup:
9673 * finds object from hkey
9674 *
9675 * input:
9676 * htblp pointer to hash table
9677 *
9678 * hkey hkey that identifies the object to be looked up
9679 *
9680 * output:
9681 * none
9682 *
9683 * return value(s):
9684 * NULL if not found
9685 * object pointer if found
9686 */
9687 static void *
9688 daplka_hash_lookup(daplka_hash_table_t *htblp, uint64_t hkey)
9689 {
9690 daplka_hash_entry_t *curr_hep;
9691 uint32_t bucket;
9692 void *objp;
9693
9694 bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9695
9696 rw_enter(&htblp->ht_table_lock, RW_READER);
9697 curr_hep = htblp->ht_buckets[bucket].hb_entries;
9698 while (curr_hep != NULL) {
9699 if (curr_hep->he_hkey == hkey) {
9700 break;
9701 }
9702 curr_hep = curr_hep->he_next;
9703 }
9704 if (curr_hep == NULL) {
9705 DERR("hash_lookup: cannot find hash entry: "
9706 "bucket %d, hkey 0x%016llx\n", bucket, (longlong_t)hkey);
9707 rw_exit(&htblp->ht_table_lock);
9708 return (NULL);
9709 }
9710 objp = curr_hep->he_objp;
9711 ASSERT(objp != NULL);
9712 if (htblp->ht_lookup_func != NULL) {
9713 (*htblp->ht_lookup_func)(objp);
9714 }
9715 rw_exit(&htblp->ht_table_lock);
9716 return (objp);
9717 }
9718
9719 /*
9720 * daplka_hash_destroy:
9721 * destroys hash table. applies free_func on all inserted objects.
9722 *
9723 * input:
9724 * htblp pointer to hash table
9725 *
9726 * output:
9727 * none
9728 *
9729 * return value(s):
9730 * none
9731 */
9732 static void
9733 daplka_hash_destroy(daplka_hash_table_t *htblp)
9734 {
9735 daplka_hash_entry_t *curr_hep, *free_hep;
9736 daplka_hash_entry_t *free_list = NULL;
9737 daplka_hash_bucket_t *hbp;
9738 uint32_t bucket, cnt, total = 0;
9739
9740 if (!htblp->ht_initialized) {
9741 DERR("hash_destroy: not initialized\n");
9742 return;
9743 }
9744 /* free all elements from hash table */
9745 rw_enter(&htblp->ht_table_lock, RW_WRITER);
9746 for (bucket = 0; bucket < htblp->ht_nbuckets; bucket++) {
9747 hbp = &htblp->ht_buckets[bucket];
9748
9749 /* build list of elements to be freed */
9750 curr_hep = hbp->hb_entries;
9751 cnt = 0;
9752 while (curr_hep != NULL) {
9753 cnt++;
9754 free_hep = curr_hep;
9755 curr_hep = curr_hep->he_next;
9756
9757 free_hep->he_next = free_list;
9758 free_list = free_hep;
9759 }
9760 ASSERT(cnt == hbp->hb_count);
9761 total += cnt;
9762 hbp->hb_count = 0;
9763 hbp->hb_entries = NULL;
9764 }
9765 ASSERT(total == htblp->ht_count);
9766 D3("hash_destroy: htblp 0x%p, nbuckets %d, freed %d hash entries\n",
9767 htblp, htblp->ht_nbuckets, total);
9768 rw_exit(&htblp->ht_table_lock);
9769
9770 /* free all objects, now without holding the hash table lock */
9771 cnt = 0;
9772 while (free_list != NULL) {
9773 cnt++;
9774 free_hep = free_list;
9775 free_list = free_list->he_next;
9776 if (htblp->ht_free_func != NULL) {
9777 (*htblp->ht_free_func)(free_hep->he_objp);
9778 }
9779 kmem_free(free_hep, sizeof (*free_hep));
9780 }
9781 ASSERT(total == cnt);
9782
9783 /* free hash buckets and destroy locks */
9784 kmem_free(htblp->ht_buckets,
9785 sizeof (daplka_hash_bucket_t) * htblp->ht_nbuckets);
9786
9787 rw_enter(&htblp->ht_table_lock, RW_WRITER);
9788 htblp->ht_buckets = NULL;
9789 htblp->ht_count = 0;
9790 htblp->ht_nbuckets = 0;
9791 htblp->ht_free_func = NULL;
9792 htblp->ht_lookup_func = NULL;
9793 htblp->ht_initialized = B_FALSE;
9794 rw_exit(&htblp->ht_table_lock);
9795
9796 mutex_destroy(&htblp->ht_key_lock);
9797 rw_destroy(&htblp->ht_table_lock);
9798 }
9799
9800 /*
9801 * daplka_hash_getsize:
9802 * return the number of objects in hash table
9803 *
9804 * input:
9805 * htblp pointer to hash table
9806 *
9807 * output:
9808 * none
9809 *
9810 * return value(s):
9811 * number of objects in hash table
9812 */
9813 static uint32_t
9814 daplka_hash_getsize(daplka_hash_table_t *htblp)
9815 {
9816 uint32_t sz;
9817
9818 rw_enter(&htblp->ht_table_lock, RW_READER);
9819 sz = htblp->ht_count;
9820 rw_exit(&htblp->ht_table_lock);
9821
9822 return (sz);
9823 }
9824
9825 /*
9826 * this function is used as ht_lookup_func above when lookup is called.
9827 * other types of objs may use a more elaborate lookup_func.
9828 */
9829 static void
9830 daplka_hash_generic_lookup(void *obj)
9831 {
9832 daplka_resource_t *rp = (daplka_resource_t *)obj;
9833
9834 mutex_enter(&rp->rs_reflock);
9835 rp->rs_refcnt++;
9836 ASSERT(rp->rs_refcnt != 0);
9837 mutex_exit(&rp->rs_reflock);
9838 }
9839
9840 /*
9841 * Generates a non-zero 32 bit hash key used for the timer hash table.
9842 */
9843 static uint32_t
9844 daplka_timer_hkey_gen()
9845 {
9846 uint32_t new_hkey;
9847
9848 do {
9849 new_hkey = atomic_inc_32_nv(&daplka_timer_hkey);
9850 } while (new_hkey == 0);
9851
9852 return (new_hkey);
9853 }
9854
9855
9856 /*
9857 * The DAPL KA debug logging routines
9858 */
9859
9860 /*
9861 * Add the string str to the end of the debug log, followed by a newline.
9862 */
9863 static void
9864 daplka_dbglog(char *str)
9865 {
9866 size_t length;
9867 size_t remlen;
9868
9869 /*
9870 * If this is the first time we've written to the log, initialize it.
9871 */
9872 if (!daplka_dbginit) {
9873 return;
9874 }
9875 mutex_enter(&daplka_dbglock);
9876 /*
9877 * Note the log is circular; if this string would run over the end,
9878 * we copy the first piece to the end and then the last piece to
9879 * the beginning of the log.
9880 */
9881 length = strlen(str);
9882
9883 remlen = (size_t)sizeof (daplka_dbgbuf) - daplka_dbgnext - 1;
9884
9885 if (length > remlen) {
9886 if (remlen)
9887 bcopy(str, daplka_dbgbuf + daplka_dbgnext, remlen);
9888 daplka_dbgbuf[sizeof (daplka_dbgbuf) - 1] = '\0';
9889 str += remlen;
9890 length -= remlen;
9891 daplka_dbgnext = 0;
9892 }
9893 bcopy(str, daplka_dbgbuf + daplka_dbgnext, length);
9894 daplka_dbgnext += length;
9895
9896 if (daplka_dbgnext >= sizeof (daplka_dbgbuf))
9897 daplka_dbgnext = 0;
9898 mutex_exit(&daplka_dbglock);
9899 }
9900
9901
9902 /*
9903 * Add a printf-style message to whichever debug logs we're currently using.
9904 */
9905 static void
9906 daplka_debug(const char *fmt, ...)
9907 {
9908 char buff[512];
9909 va_list ap;
9910 /*
9911 * The system prepends the thread id and high resolution time
9912 * (nanoseconds are dropped and so are the upper digits)
9913 * to the specified string.
9914 * The unit for timestamp is 10 microseconds.
9915 * It wraps around every 10000 seconds.
9916 * Ex: gethrtime() = X ns = X/1000 us = X/10000 10 micro sec.
9917 */
9918 int micro_time = (int)((gethrtime() / 10000) % 1000000000);
9919 (void) sprintf(buff, "th %p tm %9d: ", (void *)curthread, micro_time);
9920
9921 va_start(ap, fmt);
9922 (void) vsprintf(buff+strlen(buff), fmt, ap);
9923 va_end(ap);
9924
9925 daplka_dbglog(buff);
9926 }
9927
9928 static void
9929 daplka_console(const char *fmt, ...)
9930 {
9931 char buff[512];
9932 va_list ap;
9933
9934 va_start(ap, fmt);
9935 (void) vsprintf(buff, fmt, ap);
9936 va_end(ap);
9937
9938 cmn_err(CE_CONT, "%s", buff);
9939 }