?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dtor/input
[linux-btrfs-devel.git] / drivers / s390 / cio / qdio_main.c
blob9a122280246c1783485af4140d9785cefadca976
1 /*
2 * linux/drivers/s390/cio/qdio_main.c
3 *
4 * Linux for s390 qdio support, buffer handling, qdio API and module support.
5 *
6 * Copyright 2000,2008 IBM Corp.
7 * Author(s): Utz Bacher <utz.bacher@de.ibm.com>
8 * Jan Glauber <jang@linux.vnet.ibm.com>
9 * 2.6 cio integration by Cornelia Huck <cornelia.huck@de.ibm.com>
10 */
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/timer.h>
15 #include <linux/delay.h>
16 #include <linux/gfp.h>
17 #include <linux/io.h>
18 #include <linux/kernel_stat.h>
19 #include <linux/atomic.h>
20 #include <asm/debug.h>
21 #include <asm/qdio.h>
22
23 #include "cio.h"
24 #include "css.h"
25 #include "device.h"
26 #include "qdio.h"
27 #include "qdio_debug.h"
28
29 MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
30 "Jan Glauber <jang@linux.vnet.ibm.com>");
31 MODULE_DESCRIPTION("QDIO base support");
32 MODULE_LICENSE("GPL");
33
34 static inline int do_siga_sync(unsigned long schid,
35 unsigned int out_mask, unsigned int in_mask,
36 unsigned int fc)
37 {
38 register unsigned long __fc asm ("0") = fc;
39 register unsigned long __schid asm ("1") = schid;
40 register unsigned long out asm ("2") = out_mask;
41 register unsigned long in asm ("3") = in_mask;
42 int cc;
43
44 asm volatile(
45 " siga 0\n"
46 " ipm %0\n"
47 " srl %0,28\n"
48 : "=d" (cc)
49 : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc");
50 return cc;
51 }
52
53 static inline int do_siga_input(unsigned long schid, unsigned int mask,
54 unsigned int fc)
55 {
56 register unsigned long __fc asm ("0") = fc;
57 register unsigned long __schid asm ("1") = schid;
58 register unsigned long __mask asm ("2") = mask;
59 int cc;
60
61 asm volatile(
62 " siga 0\n"
63 " ipm %0\n"
64 " srl %0,28\n"
65 : "=d" (cc)
66 : "d" (__fc), "d" (__schid), "d" (__mask) : "cc", "memory");
67 return cc;
68 }
69
70 /**
71 * do_siga_output - perform SIGA-w/wt function
72 * @schid: subchannel id or in case of QEBSM the subchannel token
73 * @mask: which output queues to process
74 * @bb: busy bit indicator, set only if SIGA-w/wt could not access a buffer
75 * @fc: function code to perform
76 *
77 * Returns cc or QDIO_ERROR_SIGA_ACCESS_EXCEPTION.
78 * Note: For IQDC unicast queues only the highest priority queue is processed.
79 */
80 static inline int do_siga_output(unsigned long schid, unsigned long mask,
81 unsigned int *bb, unsigned int fc,
82 unsigned long aob)
83 {
84 register unsigned long __fc asm("0") = fc;
85 register unsigned long __schid asm("1") = schid;
86 register unsigned long __mask asm("2") = mask;
87 register unsigned long __aob asm("3") = aob;
88 int cc = QDIO_ERROR_SIGA_ACCESS_EXCEPTION;
89
90 asm volatile(
91 " siga 0\n"
92 "0: ipm %0\n"
93 " srl %0,28\n"
94 "1:\n"
95 EX_TABLE(0b, 1b)
96 : "+d" (cc), "+d" (__fc), "+d" (__schid), "+d" (__mask),
97 "+d" (__aob)
98 : : "cc", "memory");
99 *bb = ((unsigned int) __fc) >> 31;
100 return cc;
101 }
102
103 static inline int qdio_check_ccq(struct qdio_q *q, unsigned int ccq)
104 {
105 /* all done or next buffer state different */
106 if (ccq == 0 || ccq == 32)
107 return 0;
108 /* not all buffers processed */
109 if (ccq == 96 || ccq == 97)
110 return 1;
111 /* notify devices immediately */
112 DBF_ERROR("%4x ccq:%3d", SCH_NO(q), ccq);
113 return -EIO;
114 }
115
116 /**
117 * qdio_do_eqbs - extract buffer states for QEBSM
118 * @q: queue to manipulate
119 * @state: state of the extracted buffers
120 * @start: buffer number to start at
121 * @count: count of buffers to examine
122 * @auto_ack: automatically acknowledge buffers
123 *
124 * Returns the number of successfully extracted equal buffer states.
125 * Stops processing if a state is different from the last buffers state.
126 */
127 static int qdio_do_eqbs(struct qdio_q *q, unsigned char *state,
128 int start, int count, int auto_ack)
129 {
130 unsigned int ccq = 0;
131 int tmp_count = count, tmp_start = start;
132 int nr = q->nr;
133 int rc;
134
135 BUG_ON(!q->irq_ptr->sch_token);
136 qperf_inc(q, eqbs);
137
138 if (!q->is_input_q)
139 nr += q->irq_ptr->nr_input_qs;
140 again:
141 ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,
142 auto_ack);
143 rc = qdio_check_ccq(q, ccq);
144
145 /* At least one buffer was processed, return and extract the remaining
146 * buffers later.
147 */
148 if ((ccq == 96) && (count != tmp_count)) {
149 qperf_inc(q, eqbs_partial);
150 return (count - tmp_count);
151 }
152
153 if (rc == 1) {
154 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);
155 goto again;
156 }
157
158 if (rc < 0) {
159 DBF_ERROR("%4x EQBS ERROR", SCH_NO(q));
160 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
161 q->handler(q->irq_ptr->cdev,
162 QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
163 0, -1, -1, q->irq_ptr->int_parm);
164 return 0;
165 }
166 return count - tmp_count;
167 }
168
169 /**
170 * qdio_do_sqbs - set buffer states for QEBSM
171 * @q: queue to manipulate
172 * @state: new state of the buffers
173 * @start: first buffer number to change
174 * @count: how many buffers to change
175 *
176 * Returns the number of successfully changed buffers.
177 * Does retrying until the specified count of buffer states is set or an
178 * error occurs.
179 */
180 static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start,
181 int count)
182 {
183 unsigned int ccq = 0;
184 int tmp_count = count, tmp_start = start;
185 int nr = q->nr;
186 int rc;
187
188 if (!count)
189 return 0;
190
191 BUG_ON(!q->irq_ptr->sch_token);
192 qperf_inc(q, sqbs);
193
194 if (!q->is_input_q)
195 nr += q->irq_ptr->nr_input_qs;
196 again:
197 ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);
198 rc = qdio_check_ccq(q, ccq);
199 if (rc == 1) {
200 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);
201 qperf_inc(q, sqbs_partial);
202 goto again;
203 }
204 if (rc < 0) {
205 DBF_ERROR("%4x SQBS ERROR", SCH_NO(q));
206 DBF_ERROR("%3d%3d%2d", count, tmp_count, nr);
207 q->handler(q->irq_ptr->cdev,
208 QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
209 0, -1, -1, q->irq_ptr->int_parm);
210 return 0;
211 }
212 WARN_ON(tmp_count);
213 return count - tmp_count;
214 }
215
216 /* returns number of examined buffers and their common state in *state */
217 static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr,
218 unsigned char *state, unsigned int count,
219 int auto_ack, int merge_pending)
220 {
221 unsigned char __state = 0;
222 int i;
223
224 BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
225 BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
226
227 if (is_qebsm(q))
228 return qdio_do_eqbs(q, state, bufnr, count, auto_ack);
229
230 for (i = 0; i < count; i++) {
231 if (!__state) {
232 __state = q->slsb.val[bufnr];
233 if (merge_pending && __state == SLSB_P_OUTPUT_PENDING)
234 __state = SLSB_P_OUTPUT_EMPTY;
235 } else if (merge_pending) {
236 if ((q->slsb.val[bufnr] & __state) != __state)
237 break;
238 } else if (q->slsb.val[bufnr] != __state)
239 break;
240 bufnr = next_buf(bufnr);
241 }
242 *state = __state;
243 return i;
244 }
245
246 static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr,
247 unsigned char *state, int auto_ack)
248 {
249 return get_buf_states(q, bufnr, state, 1, auto_ack, 0);
250 }
251
252 /* wrap-around safe setting of slsb states, returns number of changed buffers */
253 static inline int set_buf_states(struct qdio_q *q, int bufnr,
254 unsigned char state, int count)
255 {
256 int i;
257
258 BUG_ON(bufnr > QDIO_MAX_BUFFERS_MASK);
259 BUG_ON(count > QDIO_MAX_BUFFERS_PER_Q);
260
261 if (is_qebsm(q))
262 return qdio_do_sqbs(q, state, bufnr, count);
263
264 for (i = 0; i < count; i++) {
265 xchg(&q->slsb.val[bufnr], state);
266 bufnr = next_buf(bufnr);
267 }
268 return count;
269 }
270
271 static inline int set_buf_state(struct qdio_q *q, int bufnr,
272 unsigned char state)
273 {
274 return set_buf_states(q, bufnr, state, 1);
275 }
276
277 /* set slsb states to initial state */
278 void qdio_init_buf_states(struct qdio_irq *irq_ptr)
279 {
280 struct qdio_q *q;
281 int i;
282
283 for_each_input_queue(irq_ptr, q, i)
284 set_buf_states(q, 0, SLSB_P_INPUT_NOT_INIT,
285 QDIO_MAX_BUFFERS_PER_Q);
286 for_each_output_queue(irq_ptr, q, i)
287 set_buf_states(q, 0, SLSB_P_OUTPUT_NOT_INIT,
288 QDIO_MAX_BUFFERS_PER_Q);
289 }
290
291 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
292 unsigned int input)
293 {
294 unsigned long schid = *((u32 *) &q->irq_ptr->schid);
295 unsigned int fc = QDIO_SIGA_SYNC;
296 int cc;
297
298 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
299 qperf_inc(q, siga_sync);
300
301 if (is_qebsm(q)) {
302 schid = q->irq_ptr->sch_token;
303 fc |= QDIO_SIGA_QEBSM_FLAG;
304 }
305
306 cc = do_siga_sync(schid, output, input, fc);
307 if (unlikely(cc))
308 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
309 return cc;
310 }
311
312 static inline int qdio_siga_sync_q(struct qdio_q *q)
313 {
314 if (q->is_input_q)
315 return qdio_siga_sync(q, 0, q->mask);
316 else
317 return qdio_siga_sync(q, q->mask, 0);
318 }
319
320 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit,
321 unsigned long aob)
322 {
323 unsigned long schid = *((u32 *) &q->irq_ptr->schid);
324 unsigned int fc = QDIO_SIGA_WRITE;
325 u64 start_time = 0;
326 int retries = 0, cc;
327 unsigned long laob = 0;
328
329 if (q->u.out.use_cq && aob != 0) {
330 fc = QDIO_SIGA_WRITEQ;
331 laob = aob;
332 }
333
334 if (is_qebsm(q)) {
335 schid = q->irq_ptr->sch_token;
336 fc |= QDIO_SIGA_QEBSM_FLAG;
337 }
338 again:
339 WARN_ON_ONCE((aob && queue_type(q) != QDIO_IQDIO_QFMT) ||
340 (aob && fc != QDIO_SIGA_WRITEQ));
341 cc = do_siga_output(schid, q->mask, busy_bit, fc, laob);
342
343 /* hipersocket busy condition */
344 if (unlikely(*busy_bit)) {
345 WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
346 retries++;
347
348 if (!start_time) {
349 start_time = get_clock();
350 goto again;
351 }
352 if ((get_clock() - start_time) < QDIO_BUSY_BIT_PATIENCE)
353 goto again;
354 }
355 if (retries) {
356 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,
357 "%4x cc2 BB1:%1d", SCH_NO(q), q->nr);
358 DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);
359 }
360 return cc;
361 }
362
363 static inline int qdio_siga_input(struct qdio_q *q)
364 {
365 unsigned long schid = *((u32 *) &q->irq_ptr->schid);
366 unsigned int fc = QDIO_SIGA_READ;
367 int cc;
368
369 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
370 qperf_inc(q, siga_read);
371
372 if (is_qebsm(q)) {
373 schid = q->irq_ptr->sch_token;
374 fc |= QDIO_SIGA_QEBSM_FLAG;
375 }
376
377 cc = do_siga_input(schid, q->mask, fc);
378 if (unlikely(cc))
379 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
380 return cc;
381 }
382
383 #define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0)
384 #define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U)
385
386 static inline void qdio_sync_queues(struct qdio_q *q)
387 {
388 /* PCI capable outbound queues will also be scanned so sync them too */
389 if (pci_out_supported(q))
390 qdio_siga_sync_all(q);
391 else
392 qdio_siga_sync_q(q);
393 }
394
395 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
396 unsigned char *state)
397 {
398 if (need_siga_sync(q))
399 qdio_siga_sync_q(q);
400 return get_buf_states(q, bufnr, state, 1, 0, 0);
401 }
402
403 static inline void qdio_stop_polling(struct qdio_q *q)
404 {
405 if (!q->u.in.polling)
406 return;
407
408 q->u.in.polling = 0;
409 qperf_inc(q, stop_polling);
410
411 /* show the card that we are not polling anymore */
412 if (is_qebsm(q)) {
413 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
414 q->u.in.ack_count);
415 q->u.in.ack_count = 0;
416 } else
417 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
418 }
419
420 static inline void account_sbals(struct qdio_q *q, int count)
421 {
422 int pos = 0;
423
424 q->q_stats.nr_sbal_total += count;
425 if (count == QDIO_MAX_BUFFERS_MASK) {
426 q->q_stats.nr_sbals[7]++;
427 return;
428 }
429 while (count >>= 1)
430 pos++;
431 q->q_stats.nr_sbals[pos]++;
432 }
433
434 static void process_buffer_error(struct qdio_q *q, int count)
435 {
436 unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :
437 SLSB_P_OUTPUT_NOT_INIT;
438
439 q->qdio_error |= QDIO_ERROR_SLSB_STATE;
440
441 /* special handling for no target buffer empty */
442 if ((!q->is_input_q &&
443 (q->sbal[q->first_to_check]->element[15].sflags) == 0x10)) {
444 qperf_inc(q, target_full);
445 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
446 q->first_to_check);
447 return;
448 }
449
450 DBF_ERROR("%4x BUF ERROR", SCH_NO(q));
451 DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
452 DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
453 DBF_ERROR("F14:%2x F15:%2x",
454 q->sbal[q->first_to_check]->element[14].sflags,
455 q->sbal[q->first_to_check]->element[15].sflags);
456
457 /*
458 * Interrupts may be avoided as long as the error is present
459 * so change the buffer state immediately to avoid starvation.
460 */
461 set_buf_states(q, q->first_to_check, state, count);
462 }
463
464 static inline void inbound_primed(struct qdio_q *q, int count)
465 {
466 int new;
467
468 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim: %02x", count);
469
470 /* for QEBSM the ACK was already set by EQBS */
471 if (is_qebsm(q)) {
472 if (!q->u.in.polling) {
473 q->u.in.polling = 1;
474 q->u.in.ack_count = count;
475 q->u.in.ack_start = q->first_to_check;
476 return;
477 }
478
479 /* delete the previous ACK's */
480 set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT,
481 q->u.in.ack_count);
482 q->u.in.ack_count = count;
483 q->u.in.ack_start = q->first_to_check;
484 return;
485 }
486
487 /*
488 * ACK the newest buffer. The ACK will be removed in qdio_stop_polling
489 * or by the next inbound run.
490 */
491 new = add_buf(q->first_to_check, count - 1);
492 if (q->u.in.polling) {
493 /* reset the previous ACK but first set the new one */
494 set_buf_state(q, new, SLSB_P_INPUT_ACK);
495 set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT);
496 } else {
497 q->u.in.polling = 1;
498 set_buf_state(q, new, SLSB_P_INPUT_ACK);
499 }
500
501 q->u.in.ack_start = new;
502 count--;
503 if (!count)
504 return;
505 /* need to change ALL buffers to get more interrupts */
506 set_buf_states(q, q->first_to_check, SLSB_P_INPUT_NOT_INIT, count);
507 }
508
509 static int get_inbound_buffer_frontier(struct qdio_q *q)
510 {
511 int count, stop;
512 unsigned char state = 0;
513
514 /*
515 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
516 * would return 0.
517 */
518 count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
519 stop = add_buf(q->first_to_check, count);
520
521 if (q->first_to_check == stop)
522 goto out;
523
524 /*
525 * No siga sync here, as a PCI or we after a thin interrupt
526 * already sync'ed the queues.
527 */
528 count = get_buf_states(q, q->first_to_check, &state, count, 1, 0);
529 if (!count)
530 goto out;
531
532 switch (state) {
533 case SLSB_P_INPUT_PRIMED:
534 inbound_primed(q, count);
535 q->first_to_check = add_buf(q->first_to_check, count);
536 if (atomic_sub(count, &q->nr_buf_used) == 0)
537 qperf_inc(q, inbound_queue_full);
538 if (q->irq_ptr->perf_stat_enabled)
539 account_sbals(q, count);
540 break;
541 case SLSB_P_INPUT_ERROR:
542 process_buffer_error(q, count);
543 q->first_to_check = add_buf(q->first_to_check, count);
544 atomic_sub(count, &q->nr_buf_used);
545 if (q->irq_ptr->perf_stat_enabled)
546 account_sbals_error(q, count);
547 break;
548 case SLSB_CU_INPUT_EMPTY:
549 case SLSB_P_INPUT_NOT_INIT:
550 case SLSB_P_INPUT_ACK:
551 if (q->irq_ptr->perf_stat_enabled)
552 q->q_stats.nr_sbal_nop++;
553 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop");
554 break;
555 default:
556 BUG();
557 }
558 out:
559 return q->first_to_check;
560 }
561
562 static int qdio_inbound_q_moved(struct qdio_q *q)
563 {
564 int bufnr;
565
566 bufnr = get_inbound_buffer_frontier(q);
567
568 if ((bufnr != q->last_move) || q->qdio_error) {
569 q->last_move = bufnr;
570 if (!is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR)
571 q->u.in.timestamp = get_clock();
572 return 1;
573 } else
574 return 0;
575 }
576
577 static inline int qdio_inbound_q_done(struct qdio_q *q)
578 {
579 unsigned char state = 0;
580
581 if (!atomic_read(&q->nr_buf_used))
582 return 1;
583
584 if (need_siga_sync(q))
585 qdio_siga_sync_q(q);
586 get_buf_state(q, q->first_to_check, &state, 0);
587
588 if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
589 /* more work coming */
590 return 0;
591
592 if (is_thinint_irq(q->irq_ptr))
593 return 1;
594
595 /* don't poll under z/VM */
596 if (MACHINE_IS_VM)
597 return 1;
598
599 /*
600 * At this point we know, that inbound first_to_check
601 * has (probably) not moved (see qdio_inbound_processing).
602 */
603 if (get_clock() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) {
604 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x",
605 q->first_to_check);
606 return 1;
607 } else
608 return 0;
609 }
610
611 static inline int contains_aobs(struct qdio_q *q)
612 {
613 return !q->is_input_q && q->u.out.use_cq;
614 }
615
616 static inline void qdio_trace_aob(struct qdio_irq *irq, struct qdio_q *q,
617 int i, struct qaob *aob)
618 {
619 int tmp;
620
621 DBF_DEV_EVENT(DBF_INFO, irq, "AOB%d:%lx", i,
622 (unsigned long) virt_to_phys(aob));
623 DBF_DEV_EVENT(DBF_INFO, irq, "RES00:%lx",
624 (unsigned long) aob->res0[0]);
625 DBF_DEV_EVENT(DBF_INFO, irq, "RES01:%lx",
626 (unsigned long) aob->res0[1]);
627 DBF_DEV_EVENT(DBF_INFO, irq, "RES02:%lx",
628 (unsigned long) aob->res0[2]);
629 DBF_DEV_EVENT(DBF_INFO, irq, "RES03:%lx",
630 (unsigned long) aob->res0[3]);
631 DBF_DEV_EVENT(DBF_INFO, irq, "RES04:%lx",
632 (unsigned long) aob->res0[4]);
633 DBF_DEV_EVENT(DBF_INFO, irq, "RES05:%lx",
634 (unsigned long) aob->res0[5]);
635 DBF_DEV_EVENT(DBF_INFO, irq, "RES1:%x", aob->res1);
636 DBF_DEV_EVENT(DBF_INFO, irq, "RES2:%x", aob->res2);
637 DBF_DEV_EVENT(DBF_INFO, irq, "RES3:%x", aob->res3);
638 DBF_DEV_EVENT(DBF_INFO, irq, "AORC:%u", aob->aorc);
639 DBF_DEV_EVENT(DBF_INFO, irq, "FLAGS:%u", aob->flags);
640 DBF_DEV_EVENT(DBF_INFO, irq, "CBTBS:%u", aob->cbtbs);
641 DBF_DEV_EVENT(DBF_INFO, irq, "SBC:%u", aob->sb_count);
642 for (tmp = 0; tmp < QDIO_MAX_ELEMENTS_PER_BUFFER; ++tmp) {
643 DBF_DEV_EVENT(DBF_INFO, irq, "SBA%d:%lx", tmp,
644 (unsigned long) aob->sba[tmp]);
645 DBF_DEV_EVENT(DBF_INFO, irq, "rSBA%d:%lx", tmp,
646 (unsigned long) q->sbal[i]->element[tmp].addr);
647 DBF_DEV_EVENT(DBF_INFO, irq, "DC%d:%u", tmp, aob->dcount[tmp]);
648 DBF_DEV_EVENT(DBF_INFO, irq, "rDC%d:%u", tmp,
649 q->sbal[i]->element[tmp].length);
650 }
651 DBF_DEV_EVENT(DBF_INFO, irq, "USER0:%lx", (unsigned long) aob->user0);
652 for (tmp = 0; tmp < 2; ++tmp) {
653 DBF_DEV_EVENT(DBF_INFO, irq, "RES4%d:%lx", tmp,
654 (unsigned long) aob->res4[tmp]);
655 }
656 DBF_DEV_EVENT(DBF_INFO, irq, "USER1:%lx", (unsigned long) aob->user1);
657 DBF_DEV_EVENT(DBF_INFO, irq, "USER2:%lx", (unsigned long) aob->user2);
658 }
659
660 static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count)
661 {
662 unsigned char state = 0;
663 int j, b = start;
664
665 if (!contains_aobs(q))
666 return;
667
668 for (j = 0; j < count; ++j) {
669 get_buf_state(q, b, &state, 0);
670 if (state == SLSB_P_OUTPUT_PENDING) {
671 struct qaob *aob = q->u.out.aobs[b];
672 if (aob == NULL)
673 continue;
674
675 BUG_ON(q->u.out.sbal_state == NULL);
676 q->u.out.sbal_state[b].flags |=
677 QDIO_OUTBUF_STATE_FLAG_PENDING;
678 q->u.out.aobs[b] = NULL;
679 } else if (state == SLSB_P_OUTPUT_EMPTY) {
680 BUG_ON(q->u.out.sbal_state == NULL);
681 q->u.out.sbal_state[b].aob = NULL;
682 }
683 b = next_buf(b);
684 }
685 }
686
687 static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q,
688 int bufnr)
689 {
690 unsigned long phys_aob = 0;
691
692 if (!q->use_cq)
693 goto out;
694
695 if (!q->aobs[bufnr]) {
696 struct qaob *aob = qdio_allocate_aob();
697 q->aobs[bufnr] = aob;
698 }
699 if (q->aobs[bufnr]) {
700 BUG_ON(q->sbal_state == NULL);
701 q->sbal_state[bufnr].flags = QDIO_OUTBUF_STATE_FLAG_NONE;
702 q->sbal_state[bufnr].aob = q->aobs[bufnr];
703 q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user;
704 phys_aob = virt_to_phys(q->aobs[bufnr]);
705 BUG_ON(phys_aob & 0xFF);
706 }
707
708 out:
709 return phys_aob;
710 }
711
712 static void qdio_kick_handler(struct qdio_q *q)
713 {
714 int start = q->first_to_kick;
715 int end = q->first_to_check;
716 int count;
717
718 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
719 return;
720
721 count = sub_buf(end, start);
722
723 if (q->is_input_q) {
724 qperf_inc(q, inbound_handler);
725 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count);
726 } else {
727 qperf_inc(q, outbound_handler);
728 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x",
729 start, count);
730 }
731
732 qdio_handle_aobs(q, start, count);
733
734 q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count,
735 q->irq_ptr->int_parm);
736
737 /* for the next time */
738 q->first_to_kick = end;
739 q->qdio_error = 0;
740 }
741
742 static void __qdio_inbound_processing(struct qdio_q *q)
743 {
744 qperf_inc(q, tasklet_inbound);
745
746 if (!qdio_inbound_q_moved(q))
747 return;
748
749 qdio_kick_handler(q);
750
751 if (!qdio_inbound_q_done(q)) {
752 /* means poll time is not yet over */
753 qperf_inc(q, tasklet_inbound_resched);
754 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
755 tasklet_schedule(&q->tasklet);
756 return;
757 }
758 }
759
760 qdio_stop_polling(q);
761 /*
762 * We need to check again to not lose initiative after
763 * resetting the ACK state.
764 */
765 if (!qdio_inbound_q_done(q)) {
766 qperf_inc(q, tasklet_inbound_resched2);
767 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
768 tasklet_schedule(&q->tasklet);
769 }
770 }
771
772 void qdio_inbound_processing(unsigned long data)
773 {
774 struct qdio_q *q = (struct qdio_q *)data;
775 __qdio_inbound_processing(q);
776 }
777
778 static int get_outbound_buffer_frontier(struct qdio_q *q)
779 {
780 int count, stop;
781 unsigned char state = 0;
782
783 if (need_siga_sync(q))
784 if (((queue_type(q) != QDIO_IQDIO_QFMT) &&
785 !pci_out_supported(q)) ||
786 (queue_type(q) == QDIO_IQDIO_QFMT &&
787 multicast_outbound(q)))
788 qdio_siga_sync_q(q);
789
790 /*
791 * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
792 * would return 0.
793 */
794 count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK);
795 stop = add_buf(q->first_to_check, count);
796 if (q->first_to_check == stop)
797 goto out;
798
799 count = get_buf_states(q, q->first_to_check, &state, count, 0, 1);
800 if (!count)
801 goto out;
802
803 switch (state) {
804 case SLSB_P_OUTPUT_PENDING:
805 BUG();
806 case SLSB_P_OUTPUT_EMPTY:
807 /* the adapter got it */
808 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,
809 "out empty:%1d %02x", q->nr, count);
810
811 atomic_sub(count, &q->nr_buf_used);
812 q->first_to_check = add_buf(q->first_to_check, count);
813 if (q->irq_ptr->perf_stat_enabled)
814 account_sbals(q, count);
815
816 break;
817 case SLSB_P_OUTPUT_ERROR:
818 process_buffer_error(q, count);
819 q->first_to_check = add_buf(q->first_to_check, count);
820 atomic_sub(count, &q->nr_buf_used);
821 if (q->irq_ptr->perf_stat_enabled)
822 account_sbals_error(q, count);
823 break;
824 case SLSB_CU_OUTPUT_PRIMED:
825 /* the adapter has not fetched the output yet */
826 if (q->irq_ptr->perf_stat_enabled)
827 q->q_stats.nr_sbal_nop++;
828 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",
829 q->nr);
830 break;
831 case SLSB_P_OUTPUT_NOT_INIT:
832 case SLSB_P_OUTPUT_HALTED:
833 break;
834 default:
835 BUG();
836 }
837
838 out:
839 return q->first_to_check;
840 }
841
842 /* all buffers processed? */
843 static inline int qdio_outbound_q_done(struct qdio_q *q)
844 {
845 return atomic_read(&q->nr_buf_used) == 0;
846 }
847
848 static inline int qdio_outbound_q_moved(struct qdio_q *q)
849 {
850 int bufnr;
851
852 bufnr = get_outbound_buffer_frontier(q);
853
854 if ((bufnr != q->last_move) || q->qdio_error) {
855 q->last_move = bufnr;
856 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr);
857 return 1;
858 } else
859 return 0;
860 }
861
862 static int qdio_kick_outbound_q(struct qdio_q *q, unsigned long aob)
863 {
864 int retries = 0, cc;
865 unsigned int busy_bit;
866
867 if (!need_siga_out(q))
868 return 0;
869
870 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);
871 retry:
872 qperf_inc(q, siga_write);
873
874 cc = qdio_siga_output(q, &busy_bit, aob);
875 switch (cc) {
876 case 0:
877 break;
878 case 2:
879 if (busy_bit) {
880 while (++retries < QDIO_BUSY_BIT_RETRIES) {
881 mdelay(QDIO_BUSY_BIT_RETRY_DELAY);
882 goto retry;
883 }
884 DBF_ERROR("%4x cc2 BBC:%1d", SCH_NO(q), q->nr);
885 cc |= QDIO_ERROR_SIGA_BUSY;
886 } else
887 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);
888 break;
889 case 1:
890 case 3:
891 DBF_ERROR("%4x SIGA-W:%1d", SCH_NO(q), cc);
892 break;
893 }
894 if (retries) {
895 DBF_ERROR("%4x cc2 BB2:%1d", SCH_NO(q), q->nr);
896 DBF_ERROR("count:%u", retries);
897 }
898 return cc;
899 }
900
901 static void __qdio_outbound_processing(struct qdio_q *q)
902 {
903 qperf_inc(q, tasklet_outbound);
904 BUG_ON(atomic_read(&q->nr_buf_used) < 0);
905
906 if (qdio_outbound_q_moved(q))
907 qdio_kick_handler(q);
908
909 if (queue_type(q) == QDIO_ZFCP_QFMT)
910 if (!pci_out_supported(q) && !qdio_outbound_q_done(q))
911 goto sched;
912
913 /* bail out for HiperSockets unicast queues */
914 if (queue_type(q) == QDIO_IQDIO_QFMT && !multicast_outbound(q))
915 return;
916
917 if ((queue_type(q) == QDIO_IQDIO_QFMT) &&
918 (atomic_read(&q->nr_buf_used)) > QDIO_IQDIO_POLL_LVL)
919 goto sched;
920
921 if (q->u.out.pci_out_enabled)
922 return;
923
924 /*
925 * Now we know that queue type is either qeth without pci enabled
926 * or HiperSockets multicast. Make sure buffer switch from PRIMED to
927 * EMPTY is noticed and outbound_handler is called after some time.
928 */
929 if (qdio_outbound_q_done(q))
930 del_timer(&q->u.out.timer);
931 else
932 if (!timer_pending(&q->u.out.timer))
933 mod_timer(&q->u.out.timer, jiffies + 10 * HZ);
934 return;
935
936 sched:
937 if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
938 return;
939 tasklet_schedule(&q->tasklet);
940 }
941
942 /* outbound tasklet */
943 void qdio_outbound_processing(unsigned long data)
944 {
945 struct qdio_q *q = (struct qdio_q *)data;
946 __qdio_outbound_processing(q);
947 }
948
949 void qdio_outbound_timer(unsigned long data)
950 {
951 struct qdio_q *q = (struct qdio_q *)data;
952
953 if (unlikely(q->irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
954 return;
955 tasklet_schedule(&q->tasklet);
956 }
957
958 static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
959 {
960 struct qdio_q *out;
961 int i;
962
963 if (!pci_out_supported(q))
964 return;
965
966 for_each_output_queue(q->irq_ptr, out, i)
967 if (!qdio_outbound_q_done(out))
968 tasklet_schedule(&out->tasklet);
969 }
970
971 static void __tiqdio_inbound_processing(struct qdio_q *q)
972 {
973 qperf_inc(q, tasklet_inbound);
974 if (need_siga_sync(q) && need_siga_sync_after_ai(q))
975 qdio_sync_queues(q);
976
977 /*
978 * The interrupt could be caused by a PCI request. Check the
979 * PCI capable outbound queues.
980 */
981 qdio_check_outbound_after_thinint(q);
982
983 if (!qdio_inbound_q_moved(q))
984 return;
985
986 qdio_kick_handler(q);
987
988 if (!qdio_inbound_q_done(q)) {
989 qperf_inc(q, tasklet_inbound_resched);
990 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED)) {
991 tasklet_schedule(&q->tasklet);
992 return;
993 }
994 }
995
996 qdio_stop_polling(q);
997 /*
998 * We need to check again to not lose initiative after
999 * resetting the ACK state.
1000 */
1001 if (!qdio_inbound_q_done(q)) {
1002 qperf_inc(q, tasklet_inbound_resched2);
1003 if (likely(q->irq_ptr->state != QDIO_IRQ_STATE_STOPPED))
1004 tasklet_schedule(&q->tasklet);
1005 }
1006 }
1007
1008 void tiqdio_inbound_processing(unsigned long data)
1009 {
1010 struct qdio_q *q = (struct qdio_q *)data;
1011 __tiqdio_inbound_processing(q);
1012 }
1013
1014 static inline void qdio_set_state(struct qdio_irq *irq_ptr,
1015 enum qdio_irq_states state)
1016 {
1017 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);
1018
1019 irq_ptr->state = state;
1020 mb();
1021 }
1022
1023 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)
1024 {
1025 if (irb->esw.esw0.erw.cons) {
1026 DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);
1027 DBF_ERROR_HEX(irb, 64);
1028 DBF_ERROR_HEX(irb->ecw, 64);
1029 }
1030 }
1031
1032 /* PCI interrupt handler */
1033 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
1034 {
1035 int i;
1036 struct qdio_q *q;
1037
1038 if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
1039 return;
1040
1041 for_each_input_queue(irq_ptr, q, i) {
1042 if (q->u.in.queue_start_poll) {
1043 /* skip if polling is enabled or already in work */
1044 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1045 &q->u.in.queue_irq_state)) {
1046 qperf_inc(q, int_discarded);
1047 continue;
1048 }
1049 q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
1050 q->irq_ptr->int_parm);
1051 } else {
1052 tasklet_schedule(&q->tasklet);
1053 }
1054 }
1055
1056 if (!pci_out_supported(q))
1057 return;
1058
1059 for_each_output_queue(irq_ptr, q, i) {
1060 if (qdio_outbound_q_done(q))
1061 continue;
1062 if (need_siga_sync(q) && need_siga_sync_out_after_pci(q))
1063 qdio_siga_sync_q(q);
1064 tasklet_schedule(&q->tasklet);
1065 }
1066 }
1067
1068 static void qdio_handle_activate_check(struct ccw_device *cdev,
1069 unsigned long intparm, int cstat, int dstat)
1070 {
1071 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1072 struct qdio_q *q;
1073
1074 DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);
1075 DBF_ERROR("intp :%lx", intparm);
1076 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
1077
1078 if (irq_ptr->nr_input_qs) {
1079 q = irq_ptr->input_qs[0];
1080 } else if (irq_ptr->nr_output_qs) {
1081 q = irq_ptr->output_qs[0];
1082 } else {
1083 dump_stack();
1084 goto no_handler;
1085 }
1086 q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE_CHECK_CONDITION,
1087 0, -1, -1, irq_ptr->int_parm);
1088 no_handler:
1089 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1090 }
1091
1092 static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat,
1093 int dstat)
1094 {
1095 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1096
1097 DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");
1098
1099 if (cstat)
1100 goto error;
1101 if (dstat & ~(DEV_STAT_DEV_END | DEV_STAT_CHN_END))
1102 goto error;
1103 if (!(dstat & DEV_STAT_DEV_END))
1104 goto error;
1105 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);
1106 return;
1107
1108 error:
1109 DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);
1110 DBF_ERROR("ds: %2x cs:%2x", dstat, cstat);
1111 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
1112 }
1113
1114 /* qdio interrupt handler */
1115 void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
1116 struct irb *irb)
1117 {
1118 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1119 int cstat, dstat;
1120
1121 if (!intparm || !irq_ptr) {
1122 DBF_ERROR("qint:%4x", cdev->private->schid.sch_no);
1123 return;
1124 }
1125
1126 kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
1127 if (irq_ptr->perf_stat_enabled)
1128 irq_ptr->perf_stat.qdio_int++;
1129
1130 if (IS_ERR(irb)) {
1131 switch (PTR_ERR(irb)) {
1132 case -EIO:
1133 DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);
1134 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);
1135 wake_up(&cdev->private->wait_q);
1136 return;
1137 default:
1138 WARN_ON(1);
1139 return;
1140 }
1141 }
1142 qdio_irq_check_sense(irq_ptr, irb);
1143 cstat = irb->scsw.cmd.cstat;
1144 dstat = irb->scsw.cmd.dstat;
1145
1146 switch (irq_ptr->state) {
1147 case QDIO_IRQ_STATE_INACTIVE:
1148 qdio_establish_handle_irq(cdev, cstat, dstat);
1149 break;
1150 case QDIO_IRQ_STATE_CLEANUP:
1151 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1152 break;
1153 case QDIO_IRQ_STATE_ESTABLISHED:
1154 case QDIO_IRQ_STATE_ACTIVE:
1155 if (cstat & SCHN_STAT_PCI) {
1156 qdio_int_handler_pci(irq_ptr);
1157 return;
1158 }
1159 if (cstat || dstat)
1160 qdio_handle_activate_check(cdev, intparm, cstat,
1161 dstat);
1162 break;
1163 case QDIO_IRQ_STATE_STOPPED:
1164 break;
1165 default:
1166 WARN_ON(1);
1167 }
1168 wake_up(&cdev->private->wait_q);
1169 }
1170
1171 /**
1172 * qdio_get_ssqd_desc - get qdio subchannel description
1173 * @cdev: ccw device to get description for
1174 * @data: where to store the ssqd
1175 *
1176 * Returns 0 or an error code. The results of the chsc are stored in the
1177 * specified structure.
1178 */
1179 int qdio_get_ssqd_desc(struct ccw_device *cdev,
1180 struct qdio_ssqd_desc *data)
1181 {
1182
1183 if (!cdev || !cdev->private)
1184 return -EINVAL;
1185
1186 DBF_EVENT("get ssqd:%4x", cdev->private->schid.sch_no);
1187 return qdio_setup_get_ssqd(NULL, &cdev->private->schid, data);
1188 }
1189 EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc);
1190
1191 static void qdio_shutdown_queues(struct ccw_device *cdev)
1192 {
1193 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1194 struct qdio_q *q;
1195 int i;
1196
1197 for_each_input_queue(irq_ptr, q, i)
1198 tasklet_kill(&q->tasklet);
1199
1200 for_each_output_queue(irq_ptr, q, i) {
1201 del_timer(&q->u.out.timer);
1202 tasklet_kill(&q->tasklet);
1203 }
1204 }
1205
1206 /**
1207 * qdio_shutdown - shut down a qdio subchannel
1208 * @cdev: associated ccw device
1209 * @how: use halt or clear to shutdown
1210 */
1211 int qdio_shutdown(struct ccw_device *cdev, int how)
1212 {
1213 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1214 int rc;
1215 unsigned long flags;
1216
1217 if (!irq_ptr)
1218 return -ENODEV;
1219
1220 BUG_ON(irqs_disabled());
1221 DBF_EVENT("qshutdown:%4x", cdev->private->schid.sch_no);
1222
1223 mutex_lock(&irq_ptr->setup_mutex);
1224 /*
1225 * Subchannel was already shot down. We cannot prevent being called
1226 * twice since cio may trigger a shutdown asynchronously.
1227 */
1228 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1229 mutex_unlock(&irq_ptr->setup_mutex);
1230 return 0;
1231 }
1232
1233 /*
1234 * Indicate that the device is going down. Scheduling the queue
1235 * tasklets is forbidden from here on.
1236 */
1237 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);
1238
1239 tiqdio_remove_input_queues(irq_ptr);
1240 qdio_shutdown_queues(cdev);
1241 qdio_shutdown_debug_entries(irq_ptr, cdev);
1242
1243 /* cleanup subchannel */
1244 spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1245
1246 if (how & QDIO_FLAG_CLEANUP_USING_CLEAR)
1247 rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP);
1248 else
1249 /* default behaviour is halt */
1250 rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP);
1251 if (rc) {
1252 DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no);
1253 DBF_ERROR("rc:%4d", rc);
1254 goto no_cleanup;
1255 }
1256
1257 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP);
1258 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1259 wait_event_interruptible_timeout(cdev->private->wait_q,
1260 irq_ptr->state == QDIO_IRQ_STATE_INACTIVE ||
1261 irq_ptr->state == QDIO_IRQ_STATE_ERR,
1262 10 * HZ);
1263 spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
1264
1265 no_cleanup:
1266 qdio_shutdown_thinint(irq_ptr);
1267
1268 /* restore interrupt handler */
1269 if ((void *)cdev->handler == (void *)qdio_int_handler)
1270 cdev->handler = irq_ptr->orig_handler;
1271 spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
1272
1273 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1274 mutex_unlock(&irq_ptr->setup_mutex);
1275 if (rc)
1276 return rc;
1277 return 0;
1278 }
1279 EXPORT_SYMBOL_GPL(qdio_shutdown);
1280
1281 /**
1282 * qdio_free - free data structures for a qdio subchannel
1283 * @cdev: associated ccw device
1284 */
1285 int qdio_free(struct ccw_device *cdev)
1286 {
1287 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1288
1289 if (!irq_ptr)
1290 return -ENODEV;
1291
1292 DBF_EVENT("qfree:%4x", cdev->private->schid.sch_no);
1293 mutex_lock(&irq_ptr->setup_mutex);
1294
1295 if (irq_ptr->debug_area != NULL) {
1296 debug_unregister(irq_ptr->debug_area);
1297 irq_ptr->debug_area = NULL;
1298 }
1299 cdev->private->qdio_data = NULL;
1300 mutex_unlock(&irq_ptr->setup_mutex);
1301
1302 qdio_release_memory(irq_ptr);
1303 return 0;
1304 }
1305 EXPORT_SYMBOL_GPL(qdio_free);
1306
1307 /**
1308 * qdio_allocate - allocate qdio queues and associated data
1309 * @init_data: initialization data
1310 */
1311 int qdio_allocate(struct qdio_initialize *init_data)
1312 {
1313 struct qdio_irq *irq_ptr;
1314
1315 DBF_EVENT("qallocate:%4x", init_data->cdev->private->schid.sch_no);
1316
1317 if ((init_data->no_input_qs && !init_data->input_handler) ||
1318 (init_data->no_output_qs && !init_data->output_handler))
1319 return -EINVAL;
1320
1321 if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) ||
1322 (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ))
1323 return -EINVAL;
1324
1325 if ((!init_data->input_sbal_addr_array) ||
1326 (!init_data->output_sbal_addr_array))
1327 return -EINVAL;
1328
1329 /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */
1330 irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1331 if (!irq_ptr)
1332 goto out_err;
1333
1334 mutex_init(&irq_ptr->setup_mutex);
1335 qdio_allocate_dbf(init_data, irq_ptr);
1336
1337 /*
1338 * Allocate a page for the chsc calls in qdio_establish.
1339 * Must be pre-allocated since a zfcp recovery will call
1340 * qdio_establish. In case of low memory and swap on a zfcp disk
1341 * we may not be able to allocate memory otherwise.
1342 */
1343 irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);
1344 if (!irq_ptr->chsc_page)
1345 goto out_rel;
1346
1347 /* qdr is used in ccw1.cda which is u32 */
1348 irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1349 if (!irq_ptr->qdr)
1350 goto out_rel;
1351 WARN_ON((unsigned long)irq_ptr->qdr & 0xfff);
1352
1353 if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs,
1354 init_data->no_output_qs))
1355 goto out_rel;
1356
1357 init_data->cdev->private->qdio_data = irq_ptr;
1358 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);
1359 return 0;
1360 out_rel:
1361 qdio_release_memory(irq_ptr);
1362 out_err:
1363 return -ENOMEM;
1364 }
1365 EXPORT_SYMBOL_GPL(qdio_allocate);
1366
1367 static void qdio_detect_hsicq(struct qdio_irq *irq_ptr)
1368 {
1369 struct qdio_q *q = irq_ptr->input_qs[0];
1370 int i, use_cq = 0;
1371
1372 if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT)
1373 use_cq = 1;
1374
1375 for_each_output_queue(irq_ptr, q, i) {
1376 if (use_cq) {
1377 if (qdio_enable_async_operation(&q->u.out) < 0) {
1378 use_cq = 0;
1379 continue;
1380 }
1381 } else
1382 qdio_disable_async_operation(&q->u.out);
1383 }
1384 DBF_EVENT("use_cq:%d", use_cq);
1385 }
1386
1387 /**
1388 * qdio_establish - establish queues on a qdio subchannel
1389 * @init_data: initialization data
1390 */
1391 int qdio_establish(struct qdio_initialize *init_data)
1392 {
1393 struct qdio_irq *irq_ptr;
1394 struct ccw_device *cdev = init_data->cdev;
1395 unsigned long saveflags;
1396 int rc;
1397
1398 DBF_EVENT("qestablish:%4x", cdev->private->schid.sch_no);
1399
1400 irq_ptr = cdev->private->qdio_data;
1401 if (!irq_ptr)
1402 return -ENODEV;
1403
1404 if (cdev->private->state != DEV_STATE_ONLINE)
1405 return -EINVAL;
1406
1407 mutex_lock(&irq_ptr->setup_mutex);
1408 qdio_setup_irq(init_data);
1409
1410 rc = qdio_establish_thinint(irq_ptr);
1411 if (rc) {
1412 mutex_unlock(&irq_ptr->setup_mutex);
1413 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1414 return rc;
1415 }
1416
1417 /* establish q */
1418 irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd;
1419 irq_ptr->ccw.flags = CCW_FLAG_SLI;
1420 irq_ptr->ccw.count = irq_ptr->equeue.count;
1421 irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr);
1422
1423 spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1424 ccw_device_set_options_mask(cdev, 0);
1425
1426 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);
1427 if (rc) {
1428 DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);
1429 DBF_ERROR("rc:%4x", rc);
1430 }
1431 spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1432
1433 if (rc) {
1434 mutex_unlock(&irq_ptr->setup_mutex);
1435 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1436 return rc;
1437 }
1438
1439 wait_event_interruptible_timeout(cdev->private->wait_q,
1440 irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||
1441 irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);
1442
1443 if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {
1444 mutex_unlock(&irq_ptr->setup_mutex);
1445 qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
1446 return -EIO;
1447 }
1448
1449 qdio_setup_ssqd_info(irq_ptr);
1450 DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
1451
1452 qdio_detect_hsicq(irq_ptr);
1453
1454 /* qebsm is now setup if available, initialize buffer states */
1455 qdio_init_buf_states(irq_ptr);
1456
1457 mutex_unlock(&irq_ptr->setup_mutex);
1458 qdio_print_subchannel_info(irq_ptr, cdev);
1459 qdio_setup_debug_entries(irq_ptr, cdev);
1460 return 0;
1461 }
1462 EXPORT_SYMBOL_GPL(qdio_establish);
1463
1464 /**
1465 * qdio_activate - activate queues on a qdio subchannel
1466 * @cdev: associated cdev
1467 */
1468 int qdio_activate(struct ccw_device *cdev)
1469 {
1470 struct qdio_irq *irq_ptr;
1471 int rc;
1472 unsigned long saveflags;
1473
1474 DBF_EVENT("qactivate:%4x", cdev->private->schid.sch_no);
1475
1476 irq_ptr = cdev->private->qdio_data;
1477 if (!irq_ptr)
1478 return -ENODEV;
1479
1480 if (cdev->private->state != DEV_STATE_ONLINE)
1481 return -EINVAL;
1482
1483 mutex_lock(&irq_ptr->setup_mutex);
1484 if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {
1485 rc = -EBUSY;
1486 goto out;
1487 }
1488
1489 irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd;
1490 irq_ptr->ccw.flags = CCW_FLAG_SLI;
1491 irq_ptr->ccw.count = irq_ptr->aqueue.count;
1492 irq_ptr->ccw.cda = 0;
1493
1494 spin_lock_irqsave(get_ccwdev_lock(cdev), saveflags);
1495 ccw_device_set_options(cdev, CCWDEV_REPORT_ALL);
1496
1497 rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE,
1498 0, DOIO_DENY_PREFETCH);
1499 if (rc) {
1500 DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);
1501 DBF_ERROR("rc:%4x", rc);
1502 }
1503 spin_unlock_irqrestore(get_ccwdev_lock(cdev), saveflags);
1504
1505 if (rc)
1506 goto out;
1507
1508 if (is_thinint_irq(irq_ptr))
1509 tiqdio_add_input_queues(irq_ptr);
1510
1511 /* wait for subchannel to become active */
1512 msleep(5);
1513
1514 switch (irq_ptr->state) {
1515 case QDIO_IRQ_STATE_STOPPED:
1516 case QDIO_IRQ_STATE_ERR:
1517 rc = -EIO;
1518 break;
1519 default:
1520 qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);
1521 rc = 0;
1522 }
1523 out:
1524 mutex_unlock(&irq_ptr->setup_mutex);
1525 return rc;
1526 }
1527 EXPORT_SYMBOL_GPL(qdio_activate);
1528
1529 static inline int buf_in_between(int bufnr, int start, int count)
1530 {
1531 int end = add_buf(start, count);
1532
1533 if (end > start) {
1534 if (bufnr >= start && bufnr < end)
1535 return 1;
1536 else
1537 return 0;
1538 }
1539
1540 /* wrap-around case */
1541 if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) ||
1542 (bufnr < end))
1543 return 1;
1544 else
1545 return 0;
1546 }
1547
1548 /**
1549 * handle_inbound - reset processed input buffers
1550 * @q: queue containing the buffers
1551 * @callflags: flags
1552 * @bufnr: first buffer to process
1553 * @count: how many buffers are emptied
1554 */
1555 static int handle_inbound(struct qdio_q *q, unsigned int callflags,
1556 int bufnr, int count)
1557 {
1558 int used, diff;
1559
1560 qperf_inc(q, inbound_call);
1561
1562 if (!q->u.in.polling)
1563 goto set;
1564
1565 /* protect against stop polling setting an ACK for an emptied slsb */
1566 if (count == QDIO_MAX_BUFFERS_PER_Q) {
1567 /* overwriting everything, just delete polling status */
1568 q->u.in.polling = 0;
1569 q->u.in.ack_count = 0;
1570 goto set;
1571 } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) {
1572 if (is_qebsm(q)) {
1573 /* partial overwrite, just update ack_start */
1574 diff = add_buf(bufnr, count);
1575 diff = sub_buf(diff, q->u.in.ack_start);
1576 q->u.in.ack_count -= diff;
1577 if (q->u.in.ack_count <= 0) {
1578 q->u.in.polling = 0;
1579 q->u.in.ack_count = 0;
1580 goto set;
1581 }
1582 q->u.in.ack_start = add_buf(q->u.in.ack_start, diff);
1583 }
1584 else
1585 /* the only ACK will be deleted, so stop polling */
1586 q->u.in.polling = 0;
1587 }
1588
1589 set:
1590 count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count);
1591
1592 used = atomic_add_return(count, &q->nr_buf_used) - count;
1593 BUG_ON(used + count > QDIO_MAX_BUFFERS_PER_Q);
1594
1595 if (need_siga_in(q))
1596 return qdio_siga_input(q);
1597
1598 return 0;
1599 }
1600
1601 /**
1602 * handle_outbound - process filled outbound buffers
1603 * @q: queue containing the buffers
1604 * @callflags: flags
1605 * @bufnr: first buffer to process
1606 * @count: how many buffers are filled
1607 */
1608 static int handle_outbound(struct qdio_q *q, unsigned int callflags,
1609 int bufnr, int count)
1610 {
1611 unsigned char state = 0;
1612 int used, rc = 0;
1613
1614 qperf_inc(q, outbound_call);
1615
1616 count = set_buf_states(q, bufnr, SLSB_CU_OUTPUT_PRIMED, count);
1617 used = atomic_add_return(count, &q->nr_buf_used);
1618 BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q);
1619
1620 if (used == QDIO_MAX_BUFFERS_PER_Q)
1621 qperf_inc(q, outbound_queue_full);
1622
1623 if (callflags & QDIO_FLAG_PCI_OUT) {
1624 q->u.out.pci_out_enabled = 1;
1625 qperf_inc(q, pci_request_int);
1626 } else
1627 q->u.out.pci_out_enabled = 0;
1628
1629 if (queue_type(q) == QDIO_IQDIO_QFMT) {
1630 unsigned long phys_aob = 0;
1631
1632 /* One SIGA-W per buffer required for unicast HSI */
1633 WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
1634
1635 phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr);
1636
1637 rc = qdio_kick_outbound_q(q, phys_aob);
1638 } else if (need_siga_sync(q)) {
1639 rc = qdio_siga_sync_q(q);
1640 } else {
1641 /* try to fast requeue buffers */
1642 get_buf_state(q, prev_buf(bufnr), &state, 0);
1643 if (state != SLSB_CU_OUTPUT_PRIMED)
1644 rc = qdio_kick_outbound_q(q, 0);
1645 else
1646 qperf_inc(q, fast_requeue);
1647 }
1648
1649 /* in case of SIGA errors we must process the error immediately */
1650 if (used >= q->u.out.scan_threshold || rc)
1651 tasklet_schedule(&q->tasklet);
1652 else
1653 /* free the SBALs in case of no further traffic */
1654 if (!timer_pending(&q->u.out.timer))
1655 mod_timer(&q->u.out.timer, jiffies + HZ);
1656 return rc;
1657 }
1658
1659 /**
1660 * do_QDIO - process input or output buffers
1661 * @cdev: associated ccw_device for the qdio subchannel
1662 * @callflags: input or output and special flags from the program
1663 * @q_nr: queue number
1664 * @bufnr: buffer number
1665 * @count: how many buffers to process
1666 */
1667 int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
1668 int q_nr, unsigned int bufnr, unsigned int count)
1669 {
1670 struct qdio_irq *irq_ptr;
1671
1672
1673 if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q)
1674 return -EINVAL;
1675
1676 irq_ptr = cdev->private->qdio_data;
1677 if (!irq_ptr)
1678 return -ENODEV;
1679
1680 DBF_DEV_EVENT(DBF_INFO, irq_ptr,
1681 "do%02x b:%02x c:%02x", callflags, bufnr, count);
1682
1683 if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
1684 return -EBUSY;
1685 if (!count)
1686 return 0;
1687 if (callflags & QDIO_FLAG_SYNC_INPUT)
1688 return handle_inbound(irq_ptr->input_qs[q_nr],
1689 callflags, bufnr, count);
1690 else if (callflags & QDIO_FLAG_SYNC_OUTPUT)
1691 return handle_outbound(irq_ptr->output_qs[q_nr],
1692 callflags, bufnr, count);
1693 return -EINVAL;
1694 }
1695 EXPORT_SYMBOL_GPL(do_QDIO);
1696
1697 /**
1698 * qdio_start_irq - process input buffers
1699 * @cdev: associated ccw_device for the qdio subchannel
1700 * @nr: input queue number
1701 *
1702 * Return codes
1703 * 0 - success
1704 * 1 - irqs not started since new data is available
1705 */
1706 int qdio_start_irq(struct ccw_device *cdev, int nr)
1707 {
1708 struct qdio_q *q;
1709 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1710
1711 if (!irq_ptr)
1712 return -ENODEV;
1713 q = irq_ptr->input_qs[nr];
1714
1715 WARN_ON(queue_irqs_enabled(q));
1716
1717 if (!shared_ind(q))
1718 xchg(q->irq_ptr->dsci, 0);
1719
1720 qdio_stop_polling(q);
1721 clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
1722
1723 /*
1724 * We need to check again to not lose initiative after
1725 * resetting the ACK state.
1726 */
1727 if (!shared_ind(q) && *q->irq_ptr->dsci)
1728 goto rescan;
1729 if (!qdio_inbound_q_done(q))
1730 goto rescan;
1731 return 0;
1732
1733 rescan:
1734 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1735 &q->u.in.queue_irq_state))
1736 return 0;
1737 else
1738 return 1;
1739
1740 }
1741 EXPORT_SYMBOL(qdio_start_irq);
1742
1743 /**
1744 * qdio_get_next_buffers - process input buffers
1745 * @cdev: associated ccw_device for the qdio subchannel
1746 * @nr: input queue number
1747 * @bufnr: first filled buffer number
1748 * @error: buffers are in error state
1749 *
1750 * Return codes
1751 * < 0 - error
1752 * = 0 - no new buffers found
1753 * > 0 - number of processed buffers
1754 */
1755 int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr,
1756 int *error)
1757 {
1758 struct qdio_q *q;
1759 int start, end;
1760 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1761
1762 if (!irq_ptr)
1763 return -ENODEV;
1764 q = irq_ptr->input_qs[nr];
1765 WARN_ON(queue_irqs_enabled(q));
1766
1767 /*
1768 * Cannot rely on automatic sync after interrupt since queues may
1769 * also be examined without interrupt.
1770 */
1771 if (need_siga_sync(q))
1772 qdio_sync_queues(q);
1773
1774 /* check the PCI capable outbound queues. */
1775 qdio_check_outbound_after_thinint(q);
1776
1777 if (!qdio_inbound_q_moved(q))
1778 return 0;
1779
1780 /* Note: upper-layer MUST stop processing immediately here ... */
1781 if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
1782 return -EIO;
1783
1784 start = q->first_to_kick;
1785 end = q->first_to_check;
1786 *bufnr = start;
1787 *error = q->qdio_error;
1788
1789 /* for the next time */
1790 q->first_to_kick = end;
1791 q->qdio_error = 0;
1792 return sub_buf(end, start);
1793 }
1794 EXPORT_SYMBOL(qdio_get_next_buffers);
1795
1796 /**
1797 * qdio_stop_irq - disable interrupt processing for the device
1798 * @cdev: associated ccw_device for the qdio subchannel
1799 * @nr: input queue number
1800 *
1801 * Return codes
1802 * 0 - interrupts were already disabled
1803 * 1 - interrupts successfully disabled
1804 */
1805 int qdio_stop_irq(struct ccw_device *cdev, int nr)
1806 {
1807 struct qdio_q *q;
1808 struct qdio_irq *irq_ptr = cdev->private->qdio_data;
1809
1810 if (!irq_ptr)
1811 return -ENODEV;
1812 q = irq_ptr->input_qs[nr];
1813
1814 if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
1815 &q->u.in.queue_irq_state))
1816 return 0;
1817 else
1818 return 1;
1819 }
1820 EXPORT_SYMBOL(qdio_stop_irq);
1821
1822 static int __init init_QDIO(void)
1823 {
1824 int rc;
1825
1826 rc = qdio_debug_init();
1827 if (rc)
1828 return rc;
1829 rc = qdio_setup_init();
1830 if (rc)
1831 goto out_debug;
1832 rc = tiqdio_allocate_memory();
1833 if (rc)
1834 goto out_cache;
1835 rc = tiqdio_register_thinints();
1836 if (rc)
1837 goto out_ti;
1838 return 0;
1839
1840 out_ti:
1841 tiqdio_free_memory();
1842 out_cache:
1843 qdio_setup_exit();
1844 out_debug:
1845 qdio_debug_exit();
1846 return rc;
1847 }
1848
1849 static void __exit exit_QDIO(void)
1850 {
1851 tiqdio_unregister_thinints();
1852 tiqdio_free_memory();
1853 qdio_setup_exit();
1854 qdio_debug_exit();
1855 }
1856
1857 module_init(init_QDIO);
1858 module_exit(exit_QDIO);
Linux Btrfs Development