1 // SPDX-License-Identifier: GPL-2.0
5 * Setup and helper functions to access QDIO.
7 * Copyright IBM Corp. 2002, 2010
10 #define KMSG_COMPONENT "zfcp"
11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
13 #include <linux/slab.h>
14 #include <linux/module.h>
16 #include "zfcp_qdio.h"
18 static bool enable_multibuffer
= true;
19 module_param_named(datarouter
, enable_multibuffer
, bool, 0400);
20 MODULE_PARM_DESC(datarouter
, "Enable hardware data router support (default on)");
22 static void zfcp_qdio_handler_error(struct zfcp_qdio
*qdio
, char *id
,
23 unsigned int qdio_err
)
25 struct zfcp_adapter
*adapter
= qdio
->adapter
;
27 dev_warn(&adapter
->ccw_device
->dev
, "A QDIO problem occurred\n");
29 if (qdio_err
& QDIO_ERROR_SLSB_STATE
) {
30 zfcp_qdio_siosl(adapter
);
31 zfcp_erp_adapter_shutdown(adapter
, 0, id
);
34 zfcp_erp_adapter_reopen(adapter
,
35 ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED
|
36 ZFCP_STATUS_COMMON_ERP_FAILED
, id
);
39 static void zfcp_qdio_zero_sbals(struct qdio_buffer
*sbal
[], int first
, int cnt
)
43 for (i
= first
; i
< first
+ cnt
; i
++) {
44 sbal_idx
= i
% QDIO_MAX_BUFFERS_PER_Q
;
45 memset(sbal
[sbal_idx
], 0, sizeof(struct qdio_buffer
));
49 /* this needs to be called prior to updating the queue fill level */
50 static inline void zfcp_qdio_account(struct zfcp_qdio
*qdio
)
52 unsigned long long now
, span
;
55 now
= get_tod_clock_monotonic();
56 span
= (now
- qdio
->req_q_time
) >> 12;
57 used
= QDIO_MAX_BUFFERS_PER_Q
- atomic_read(&qdio
->req_q_free
);
58 qdio
->req_q_util
+= used
* span
;
59 qdio
->req_q_time
= now
;
62 static void zfcp_qdio_int_req(struct ccw_device
*cdev
, unsigned int qdio_err
,
63 int queue_no
, int idx
, int count
,
66 struct zfcp_qdio
*qdio
= (struct zfcp_qdio
*) parm
;
68 if (unlikely(qdio_err
)) {
69 zfcp_qdio_handler_error(qdio
, "qdireq1", qdio_err
);
73 /* cleanup all SBALs being program-owned now */
74 zfcp_qdio_zero_sbals(qdio
->req_q
, idx
, count
);
76 spin_lock_irq(&qdio
->stat_lock
);
77 zfcp_qdio_account(qdio
);
78 spin_unlock_irq(&qdio
->stat_lock
);
79 atomic_add(count
, &qdio
->req_q_free
);
80 wake_up(&qdio
->req_q_wq
);
83 static void zfcp_qdio_int_resp(struct ccw_device
*cdev
, unsigned int qdio_err
,
84 int queue_no
, int idx
, int count
,
87 struct zfcp_qdio
*qdio
= (struct zfcp_qdio
*) parm
;
88 struct zfcp_adapter
*adapter
= qdio
->adapter
;
89 int sbal_no
, sbal_idx
;
91 if (unlikely(qdio_err
)) {
92 if (zfcp_adapter_multi_buffer_active(adapter
)) {
93 void *pl
[ZFCP_QDIO_MAX_SBALS_PER_REQ
+ 1];
94 struct qdio_buffer_element
*sbale
;
99 ZFCP_QDIO_MAX_SBALS_PER_REQ
* sizeof(void *));
100 sbale
= qdio
->res_q
[idx
]->element
;
101 req_id
= (u64
) sbale
->addr
;
102 scount
= min(sbale
->scount
+ 1,
103 ZFCP_QDIO_MAX_SBALS_PER_REQ
+ 1);
104 /* incl. signaling SBAL */
106 for (sbal_no
= 0; sbal_no
< scount
; sbal_no
++) {
107 sbal_idx
= (idx
+ sbal_no
) %
108 QDIO_MAX_BUFFERS_PER_Q
;
109 pl
[sbal_no
] = qdio
->res_q
[sbal_idx
];
111 zfcp_dbf_hba_def_err(adapter
, req_id
, scount
, pl
);
113 zfcp_qdio_handler_error(qdio
, "qdires1", qdio_err
);
118 * go through all SBALs from input queue currently
119 * returned by QDIO layer
121 for (sbal_no
= 0; sbal_no
< count
; sbal_no
++) {
122 sbal_idx
= (idx
+ sbal_no
) % QDIO_MAX_BUFFERS_PER_Q
;
123 /* go through all SBALEs of SBAL */
124 zfcp_fsf_reqid_check(qdio
, sbal_idx
);
128 * put SBALs back to response queue
130 if (do_QDIO(cdev
, QDIO_FLAG_SYNC_INPUT
, 0, idx
, count
))
131 zfcp_erp_adapter_reopen(qdio
->adapter
, 0, "qdires2");
134 static struct qdio_buffer_element
*
135 zfcp_qdio_sbal_chain(struct zfcp_qdio
*qdio
, struct zfcp_qdio_req
*q_req
)
137 struct qdio_buffer_element
*sbale
;
139 /* set last entry flag in current SBALE of current SBAL */
140 sbale
= zfcp_qdio_sbale_curr(qdio
, q_req
);
141 sbale
->eflags
|= SBAL_EFLAGS_LAST_ENTRY
;
143 /* don't exceed last allowed SBAL */
144 if (q_req
->sbal_last
== q_req
->sbal_limit
)
147 /* set chaining flag in first SBALE of current SBAL */
148 sbale
= zfcp_qdio_sbale_req(qdio
, q_req
);
149 sbale
->sflags
|= SBAL_SFLAGS0_MORE_SBALS
;
151 /* calculate index of next SBAL */
153 q_req
->sbal_last
%= QDIO_MAX_BUFFERS_PER_Q
;
155 /* keep this requests number of SBALs up-to-date */
156 q_req
->sbal_number
++;
157 BUG_ON(q_req
->sbal_number
> ZFCP_QDIO_MAX_SBALS_PER_REQ
);
159 /* start at first SBALE of new SBAL */
160 q_req
->sbale_curr
= 0;
162 /* set storage-block type for new SBAL */
163 sbale
= zfcp_qdio_sbale_curr(qdio
, q_req
);
164 sbale
->sflags
|= q_req
->sbtype
;
169 static struct qdio_buffer_element
*
170 zfcp_qdio_sbale_next(struct zfcp_qdio
*qdio
, struct zfcp_qdio_req
*q_req
)
172 if (q_req
->sbale_curr
== qdio
->max_sbale_per_sbal
- 1)
173 return zfcp_qdio_sbal_chain(qdio
, q_req
);
175 return zfcp_qdio_sbale_curr(qdio
, q_req
);
179 * zfcp_qdio_sbals_from_sg - fill SBALs from scatter-gather list
180 * @qdio: pointer to struct zfcp_qdio
181 * @q_req: pointer to struct zfcp_qdio_req
182 * @sg: scatter-gather list
183 * @max_sbals: upper bound for number of SBALs to be used
184 * Returns: zero or -EINVAL on error
186 int zfcp_qdio_sbals_from_sg(struct zfcp_qdio
*qdio
, struct zfcp_qdio_req
*q_req
,
187 struct scatterlist
*sg
)
189 struct qdio_buffer_element
*sbale
;
191 /* set storage-block type for this request */
192 sbale
= zfcp_qdio_sbale_req(qdio
, q_req
);
193 sbale
->sflags
|= q_req
->sbtype
;
195 for (; sg
; sg
= sg_next(sg
)) {
196 sbale
= zfcp_qdio_sbale_next(qdio
, q_req
);
198 atomic_inc(&qdio
->req_q_full
);
199 zfcp_qdio_zero_sbals(qdio
->req_q
, q_req
->sbal_first
,
203 sbale
->addr
= sg_virt(sg
);
204 sbale
->length
= sg
->length
;
209 static int zfcp_qdio_sbal_check(struct zfcp_qdio
*qdio
)
211 if (atomic_read(&qdio
->req_q_free
) ||
212 !(atomic_read(&qdio
->adapter
->status
) & ZFCP_STATUS_ADAPTER_QDIOUP
))
218 * zfcp_qdio_sbal_get - get free sbal in request queue, wait if necessary
219 * @qdio: pointer to struct zfcp_qdio
221 * The req_q_lock must be held by the caller of this function, and
222 * this function may only be called from process context; it will
223 * sleep when waiting for a free sbal.
225 * Returns: 0 on success, -EIO if there is no free sbal after waiting.
227 int zfcp_qdio_sbal_get(struct zfcp_qdio
*qdio
)
231 ret
= wait_event_interruptible_lock_irq_timeout(qdio
->req_q_wq
,
232 zfcp_qdio_sbal_check(qdio
), qdio
->req_q_lock
, 5 * HZ
);
234 if (!(atomic_read(&qdio
->adapter
->status
) & ZFCP_STATUS_ADAPTER_QDIOUP
))
241 atomic_inc(&qdio
->req_q_full
);
242 /* assume hanging outbound queue, try queue recovery */
243 zfcp_erp_adapter_reopen(qdio
->adapter
, 0, "qdsbg_1");
250 * zfcp_qdio_send - set PCI flag in first SBALE and send req to QDIO
251 * @qdio: pointer to struct zfcp_qdio
252 * @q_req: pointer to struct zfcp_qdio_req
253 * Returns: 0 on success, error otherwise
255 int zfcp_qdio_send(struct zfcp_qdio
*qdio
, struct zfcp_qdio_req
*q_req
)
258 u8 sbal_number
= q_req
->sbal_number
;
260 spin_lock(&qdio
->stat_lock
);
261 zfcp_qdio_account(qdio
);
262 spin_unlock(&qdio
->stat_lock
);
264 retval
= do_QDIO(qdio
->adapter
->ccw_device
, QDIO_FLAG_SYNC_OUTPUT
, 0,
265 q_req
->sbal_first
, sbal_number
);
267 if (unlikely(retval
)) {
268 zfcp_qdio_zero_sbals(qdio
->req_q
, q_req
->sbal_first
,
273 /* account for transferred buffers */
274 atomic_sub(sbal_number
, &qdio
->req_q_free
);
275 qdio
->req_q_idx
+= sbal_number
;
276 qdio
->req_q_idx
%= QDIO_MAX_BUFFERS_PER_Q
;
282 static void zfcp_qdio_setup_init_data(struct qdio_initialize
*id
,
283 struct zfcp_qdio
*qdio
)
285 memset(id
, 0, sizeof(*id
));
286 id
->cdev
= qdio
->adapter
->ccw_device
;
287 id
->q_format
= QDIO_ZFCP_QFMT
;
288 memcpy(id
->adapter_name
, dev_name(&id
->cdev
->dev
), 8);
289 ASCEBC(id
->adapter_name
, 8);
290 id
->qib_rflags
= QIB_RFLAGS_ENABLE_DATA_DIV
;
291 if (enable_multibuffer
)
292 id
->qdr_ac
|= QDR_AC_MULTI_BUFFER_ENABLE
;
294 id
->no_output_qs
= 1;
295 id
->input_handler
= zfcp_qdio_int_resp
;
296 id
->output_handler
= zfcp_qdio_int_req
;
297 id
->int_parm
= (unsigned long) qdio
;
298 id
->input_sbal_addr_array
= (void **) (qdio
->res_q
);
299 id
->output_sbal_addr_array
= (void **) (qdio
->req_q
);
301 QDIO_MAX_BUFFERS_PER_Q
- ZFCP_QDIO_MAX_SBALS_PER_REQ
* 2;
305 * zfcp_qdio_allocate - allocate queue memory and initialize QDIO data
306 * @adapter: pointer to struct zfcp_adapter
307 * Returns: -ENOMEM on memory allocation error or return value from
310 static int zfcp_qdio_allocate(struct zfcp_qdio
*qdio
)
312 struct qdio_initialize init_data
;
315 ret
= qdio_alloc_buffers(qdio
->req_q
, QDIO_MAX_BUFFERS_PER_Q
);
319 ret
= qdio_alloc_buffers(qdio
->res_q
, QDIO_MAX_BUFFERS_PER_Q
);
323 zfcp_qdio_setup_init_data(&init_data
, qdio
);
324 init_waitqueue_head(&qdio
->req_q_wq
);
326 ret
= qdio_allocate(&init_data
);
333 qdio_free_buffers(qdio
->res_q
, QDIO_MAX_BUFFERS_PER_Q
);
335 qdio_free_buffers(qdio
->req_q
, QDIO_MAX_BUFFERS_PER_Q
);
340 * zfcp_close_qdio - close qdio queues for an adapter
341 * @qdio: pointer to structure zfcp_qdio
343 void zfcp_qdio_close(struct zfcp_qdio
*qdio
)
345 struct zfcp_adapter
*adapter
= qdio
->adapter
;
348 if (!(atomic_read(&adapter
->status
) & ZFCP_STATUS_ADAPTER_QDIOUP
))
351 /* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */
352 spin_lock_irq(&qdio
->req_q_lock
);
353 atomic_andnot(ZFCP_STATUS_ADAPTER_QDIOUP
, &adapter
->status
);
354 spin_unlock_irq(&qdio
->req_q_lock
);
356 wake_up(&qdio
->req_q_wq
);
358 qdio_shutdown(adapter
->ccw_device
, QDIO_FLAG_CLEANUP_USING_CLEAR
);
360 /* cleanup used outbound sbals */
361 count
= atomic_read(&qdio
->req_q_free
);
362 if (count
< QDIO_MAX_BUFFERS_PER_Q
) {
363 idx
= (qdio
->req_q_idx
+ count
) % QDIO_MAX_BUFFERS_PER_Q
;
364 count
= QDIO_MAX_BUFFERS_PER_Q
- count
;
365 zfcp_qdio_zero_sbals(qdio
->req_q
, idx
, count
);
368 atomic_set(&qdio
->req_q_free
, 0);
372 * zfcp_qdio_open - prepare and initialize response queue
373 * @qdio: pointer to struct zfcp_qdio
374 * Returns: 0 on success, otherwise -EIO
376 int zfcp_qdio_open(struct zfcp_qdio
*qdio
)
378 struct qdio_buffer_element
*sbale
;
379 struct qdio_initialize init_data
;
380 struct zfcp_adapter
*adapter
= qdio
->adapter
;
381 struct ccw_device
*cdev
= adapter
->ccw_device
;
382 struct qdio_ssqd_desc ssqd
;
385 if (atomic_read(&adapter
->status
) & ZFCP_STATUS_ADAPTER_QDIOUP
)
388 atomic_andnot(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED
,
389 &qdio
->adapter
->status
);
391 zfcp_qdio_setup_init_data(&init_data
, qdio
);
393 if (qdio_establish(&init_data
))
394 goto failed_establish
;
396 if (qdio_get_ssqd_desc(init_data
.cdev
, &ssqd
))
399 if (ssqd
.qdioac2
& CHSC_AC2_DATA_DIV_ENABLED
)
400 atomic_or(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED
,
401 &qdio
->adapter
->status
);
403 if (ssqd
.qdioac2
& CHSC_AC2_MULTI_BUFFER_ENABLED
) {
404 atomic_or(ZFCP_STATUS_ADAPTER_MB_ACT
, &adapter
->status
);
405 qdio
->max_sbale_per_sbal
= QDIO_MAX_ELEMENTS_PER_BUFFER
;
407 atomic_andnot(ZFCP_STATUS_ADAPTER_MB_ACT
, &adapter
->status
);
408 qdio
->max_sbale_per_sbal
= QDIO_MAX_ELEMENTS_PER_BUFFER
- 1;
411 qdio
->max_sbale_per_req
=
412 ZFCP_QDIO_MAX_SBALS_PER_REQ
* qdio
->max_sbale_per_sbal
414 if (qdio_activate(cdev
))
417 for (cc
= 0; cc
< QDIO_MAX_BUFFERS_PER_Q
; cc
++) {
418 sbale
= &(qdio
->res_q
[cc
]->element
[0]);
420 sbale
->eflags
= SBAL_EFLAGS_LAST_ENTRY
;
425 if (do_QDIO(cdev
, QDIO_FLAG_SYNC_INPUT
, 0, 0, QDIO_MAX_BUFFERS_PER_Q
))
428 /* set index of first available SBALS / number of available SBALS */
430 atomic_set(&qdio
->req_q_free
, QDIO_MAX_BUFFERS_PER_Q
);
431 atomic_or(ZFCP_STATUS_ADAPTER_QDIOUP
, &qdio
->adapter
->status
);
433 if (adapter
->scsi_host
) {
434 adapter
->scsi_host
->sg_tablesize
= qdio
->max_sbale_per_req
;
435 adapter
->scsi_host
->max_sectors
= qdio
->max_sbale_per_req
* 8;
441 qdio_shutdown(cdev
, QDIO_FLAG_CLEANUP_USING_CLEAR
);
444 "Setting up the QDIO connection to the FCP adapter failed\n");
448 void zfcp_qdio_destroy(struct zfcp_qdio
*qdio
)
453 if (qdio
->adapter
->ccw_device
)
454 qdio_free(qdio
->adapter
->ccw_device
);
456 qdio_free_buffers(qdio
->req_q
, QDIO_MAX_BUFFERS_PER_Q
);
457 qdio_free_buffers(qdio
->res_q
, QDIO_MAX_BUFFERS_PER_Q
);
461 int zfcp_qdio_setup(struct zfcp_adapter
*adapter
)
463 struct zfcp_qdio
*qdio
;
465 qdio
= kzalloc(sizeof(struct zfcp_qdio
), GFP_KERNEL
);
469 qdio
->adapter
= adapter
;
471 if (zfcp_qdio_allocate(qdio
)) {
476 spin_lock_init(&qdio
->req_q_lock
);
477 spin_lock_init(&qdio
->stat_lock
);
479 adapter
->qdio
= qdio
;
484 * zfcp_qdio_siosl - Trigger logging in FCP channel
485 * @adapter: The zfcp_adapter where to trigger logging
487 * Call the cio siosl function to trigger hardware logging. This
488 * wrapper function sets a flag to ensure hardware logging is only
489 * triggered once before going through qdio shutdown.
491 * The triggers are always run from qdio tasklet context, so no
492 * additional synchronization is necessary.
494 void zfcp_qdio_siosl(struct zfcp_adapter
*adapter
)
498 if (atomic_read(&adapter
->status
) & ZFCP_STATUS_ADAPTER_SIOSL_ISSUED
)
501 rc
= ccw_device_siosl(adapter
->ccw_device
);
503 atomic_or(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED
,