1 // SPDX-License-Identifier: GPL-2.0
3 * S/390 common I/O routines -- low level i/o calls
5 * Copyright IBM Corp. 1999, 2008
6 * Author(s): Ingo Adlung (adlung@de.ibm.com)
7 * Cornelia Huck (cornelia.huck@de.ibm.com)
8 * Arnd Bergmann (arndb@de.ibm.com)
9 * Martin Schwidefsky (schwidefsky@de.ibm.com)
12 #define KMSG_COMPONENT "cio"
13 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
15 #include <linux/ftrace.h>
16 #include <linux/module.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/interrupt.h>
22 #include <linux/irq.h>
24 #include <asm/delay.h>
26 #include <asm/irq_regs.h>
27 #include <asm/setup.h>
29 #include <asm/chpid.h>
32 #include <linux/sched/cputime.h>
41 #include "blacklist.h"
42 #include "cio_debug.h"
46 debug_info_t
*cio_debug_msg_id
;
47 debug_info_t
*cio_debug_trace_id
;
48 debug_info_t
*cio_debug_crw_id
;
50 DEFINE_PER_CPU_ALIGNED(struct irb
, cio_irb
);
51 EXPORT_PER_CPU_SYMBOL(cio_irb
);
54 * Function: cio_debug_init
55 * Initializes three debug logs for common I/O:
56 * - cio_msg logs generic cio messages
57 * - cio_trace logs the calling of different functions
58 * - cio_crw logs machine check related cio messages
60 static int __init
cio_debug_init(void)
62 cio_debug_msg_id
= debug_register("cio_msg", 16, 1, 11 * sizeof(long));
63 if (!cio_debug_msg_id
)
65 debug_register_view(cio_debug_msg_id
, &debug_sprintf_view
);
66 debug_set_level(cio_debug_msg_id
, 2);
67 cio_debug_trace_id
= debug_register("cio_trace", 16, 1, 16);
68 if (!cio_debug_trace_id
)
70 debug_register_view(cio_debug_trace_id
, &debug_hex_ascii_view
);
71 debug_set_level(cio_debug_trace_id
, 2);
72 cio_debug_crw_id
= debug_register("cio_crw", 8, 1, 8 * sizeof(long));
73 if (!cio_debug_crw_id
)
75 debug_register_view(cio_debug_crw_id
, &debug_sprintf_view
);
76 debug_set_level(cio_debug_crw_id
, 4);
80 debug_unregister(cio_debug_msg_id
);
81 debug_unregister(cio_debug_trace_id
);
82 debug_unregister(cio_debug_crw_id
);
86 arch_initcall (cio_debug_init
);
88 int cio_set_options(struct subchannel
*sch
, int flags
)
90 struct io_subchannel_private
*priv
= to_io_private(sch
);
92 priv
->options
.suspend
= (flags
& DOIO_ALLOW_SUSPEND
) != 0;
93 priv
->options
.prefetch
= (flags
& DOIO_DENY_PREFETCH
) != 0;
94 priv
->options
.inter
= (flags
& DOIO_SUPPRESS_INTER
) != 0;
99 cio_start_handle_notoper(struct subchannel
*sch
, __u8 lpm
)
108 CIO_MSG_EVENT(2, "cio_start: 'not oper' status for "
109 "subchannel 0.%x.%04x!\n", sch
->schid
.ssid
,
112 if (cio_update_schib(sch
))
115 sprintf(dbf_text
, "no%s", dev_name(&sch
->dev
));
116 CIO_TRACE_EVENT(0, dbf_text
);
117 CIO_HEX_EVENT(0, &sch
->schib
, sizeof (struct schib
));
119 return (sch
->lpm
? -EACCES
: -ENODEV
);
123 cio_start_key (struct subchannel
*sch
, /* subchannel structure */
124 struct ccw1
* cpa
, /* logical channel prog addr */
125 __u8 lpm
, /* logical path mask */
126 __u8 key
) /* storage key */
128 struct io_subchannel_private
*priv
= to_io_private(sch
);
129 union orb
*orb
= &priv
->orb
;
132 CIO_TRACE_EVENT(5, "stIO");
133 CIO_TRACE_EVENT(5, dev_name(&sch
->dev
));
135 memset(orb
, 0, sizeof(union orb
));
136 /* sch is always under 2G. */
137 orb
->cmd
.intparm
= (u32
)(addr_t
)sch
;
140 orb
->cmd
.pfch
= priv
->options
.prefetch
== 0;
141 orb
->cmd
.spnd
= priv
->options
.suspend
;
142 orb
->cmd
.ssic
= priv
->options
.suspend
&& priv
->options
.inter
;
143 orb
->cmd
.lpm
= (lpm
!= 0) ? lpm
: sch
->lpm
;
145 * for 64 bit we always support 64 bit IDAWs with 4k page size only
149 orb
->cmd
.key
= key
>> 4;
150 /* issue "Start Subchannel" */
151 orb
->cmd
.cpa
= (__u32
) __pa(cpa
);
152 ccode
= ssch(sch
->schid
, orb
);
154 /* process condition code */
155 CIO_HEX_EVENT(5, &ccode
, sizeof(ccode
));
160 * initialize device status information
162 sch
->schib
.scsw
.cmd
.actl
|= SCSW_ACTL_START_PEND
;
164 case 1: /* status pending */
167 case 3: /* device/path not operational */
168 return cio_start_handle_notoper(sch
, lpm
);
173 EXPORT_SYMBOL_GPL(cio_start_key
);
176 cio_start (struct subchannel
*sch
, struct ccw1
*cpa
, __u8 lpm
)
178 return cio_start_key(sch
, cpa
, lpm
, PAGE_DEFAULT_KEY
);
180 EXPORT_SYMBOL_GPL(cio_start
);
183 * resume suspended I/O operation
186 cio_resume (struct subchannel
*sch
)
190 CIO_TRACE_EVENT(4, "resIO");
191 CIO_TRACE_EVENT(4, dev_name(&sch
->dev
));
193 ccode
= rsch (sch
->schid
);
195 CIO_HEX_EVENT(4, &ccode
, sizeof(ccode
));
199 sch
->schib
.scsw
.cmd
.actl
|= SCSW_ACTL_RESUME_PEND
;
207 * useless to wait for request completion
208 * as device is no longer operational !
213 EXPORT_SYMBOL_GPL(cio_resume
);
219 cio_halt(struct subchannel
*sch
)
226 CIO_TRACE_EVENT(2, "haltIO");
227 CIO_TRACE_EVENT(2, dev_name(&sch
->dev
));
230 * Issue "Halt subchannel" and process condition code
232 ccode
= hsch (sch
->schid
);
234 CIO_HEX_EVENT(2, &ccode
, sizeof(ccode
));
238 sch
->schib
.scsw
.cmd
.actl
|= SCSW_ACTL_HALT_PEND
;
240 case 1: /* status pending */
243 default: /* device not operational */
247 EXPORT_SYMBOL_GPL(cio_halt
);
250 * Clear I/O operation
253 cio_clear(struct subchannel
*sch
)
260 CIO_TRACE_EVENT(2, "clearIO");
261 CIO_TRACE_EVENT(2, dev_name(&sch
->dev
));
264 * Issue "Clear subchannel" and process condition code
266 ccode
= csch (sch
->schid
);
268 CIO_HEX_EVENT(2, &ccode
, sizeof(ccode
));
272 sch
->schib
.scsw
.cmd
.actl
|= SCSW_ACTL_CLEAR_PEND
;
274 default: /* device not operational */
278 EXPORT_SYMBOL_GPL(cio_clear
);
281 * Function: cio_cancel
282 * Issues a "Cancel Subchannel" on the specified subchannel
283 * Note: We don't need any fancy intparms and flags here
284 * since xsch is executed synchronously.
285 * Only for common I/O internal use as for now.
288 cio_cancel (struct subchannel
*sch
)
295 CIO_TRACE_EVENT(2, "cancelIO");
296 CIO_TRACE_EVENT(2, dev_name(&sch
->dev
));
298 ccode
= xsch (sch
->schid
);
300 CIO_HEX_EVENT(2, &ccode
, sizeof(ccode
));
303 case 0: /* success */
304 /* Update information in scsw. */
305 if (cio_update_schib(sch
))
308 case 1: /* status pending */
310 case 2: /* not applicable */
312 default: /* not oper */
316 EXPORT_SYMBOL_GPL(cio_cancel
);
319 * cio_cancel_halt_clear - Cancel running I/O by performing cancel, halt
320 * and clear ordinally if subchannel is valid.
321 * @sch: subchannel on which to perform the cancel_halt_clear operation
322 * @iretry: the number of the times remained to retry the next operation
324 * This should be called repeatedly since halt/clear are asynchronous
325 * operations. We do one try with cio_cancel, three tries with cio_halt,
326 * 255 tries with cio_clear. The caller should initialize @iretry with
327 * the value 255 for its first call to this, and keep using the same
328 * @iretry in the subsequent calls until it gets a non -EBUSY return.
330 * Returns 0 if device now idle, -ENODEV for device not operational,
331 * -EBUSY if an interrupt is expected (either from halt/clear or from a
332 * status pending), and -EIO if out of retries.
334 int cio_cancel_halt_clear(struct subchannel
*sch
, int *iretry
)
338 if (cio_update_schib(sch
))
340 if (!sch
->schib
.pmcw
.ena
)
341 /* Not operational -> done. */
343 /* Stage 1: cancel io. */
344 if (!(scsw_actl(&sch
->schib
.scsw
) & SCSW_ACTL_HALT_PEND
) &&
345 !(scsw_actl(&sch
->schib
.scsw
) & SCSW_ACTL_CLEAR_PEND
)) {
346 if (!scsw_is_tm(&sch
->schib
.scsw
)) {
347 ret
= cio_cancel(sch
);
352 * Cancel io unsuccessful or not applicable (transport mode).
353 * Continue with asynchronous instructions.
355 *iretry
= 3; /* 3 halt retries. */
357 /* Stage 2: halt io. */
358 if (!(scsw_actl(&sch
->schib
.scsw
) & SCSW_ACTL_CLEAR_PEND
)) {
363 return (ret
== 0) ? -EBUSY
: ret
;
365 /* Halt io unsuccessful. */
366 *iretry
= 255; /* 255 clear retries. */
368 /* Stage 3: clear io. */
371 ret
= cio_clear(sch
);
372 return (ret
== 0) ? -EBUSY
: ret
;
374 /* Function was unsuccessful */
377 EXPORT_SYMBOL_GPL(cio_cancel_halt_clear
);
379 static void cio_apply_config(struct subchannel
*sch
, struct schib
*schib
)
381 schib
->pmcw
.intparm
= sch
->config
.intparm
;
382 schib
->pmcw
.mbi
= sch
->config
.mbi
;
383 schib
->pmcw
.isc
= sch
->config
.isc
;
384 schib
->pmcw
.ena
= sch
->config
.ena
;
385 schib
->pmcw
.mme
= sch
->config
.mme
;
386 schib
->pmcw
.mp
= sch
->config
.mp
;
387 schib
->pmcw
.csense
= sch
->config
.csense
;
388 schib
->pmcw
.mbfc
= sch
->config
.mbfc
;
389 if (sch
->config
.mbfc
)
390 schib
->mba
= sch
->config
.mba
;
393 static int cio_check_config(struct subchannel
*sch
, struct schib
*schib
)
395 return (schib
->pmcw
.intparm
== sch
->config
.intparm
) &&
396 (schib
->pmcw
.mbi
== sch
->config
.mbi
) &&
397 (schib
->pmcw
.isc
== sch
->config
.isc
) &&
398 (schib
->pmcw
.ena
== sch
->config
.ena
) &&
399 (schib
->pmcw
.mme
== sch
->config
.mme
) &&
400 (schib
->pmcw
.mp
== sch
->config
.mp
) &&
401 (schib
->pmcw
.csense
== sch
->config
.csense
) &&
402 (schib
->pmcw
.mbfc
== sch
->config
.mbfc
) &&
403 (!sch
->config
.mbfc
|| (schib
->mba
== sch
->config
.mba
));
407 * cio_commit_config - apply configuration to the subchannel
409 int cio_commit_config(struct subchannel
*sch
)
411 int ccode
, retry
, ret
= 0;
415 if (stsch(sch
->schid
, &schib
) || !css_sch_is_valid(&schib
))
418 for (retry
= 0; retry
< 5; retry
++) {
419 /* copy desired changes to local schib */
420 cio_apply_config(sch
, &schib
);
421 ccode
= msch(sch
->schid
, &schib
);
422 if (ccode
< 0) /* -EIO if msch gets a program check. */
425 case 0: /* successful */
426 if (stsch(sch
->schid
, &schib
) ||
427 !css_sch_is_valid(&schib
))
429 if (cio_check_config(sch
, &schib
)) {
430 /* commit changes from local schib */
431 memcpy(&sch
->schib
, &schib
, sizeof(schib
));
436 case 1: /* status pending */
438 if (tsch(sch
->schid
, &irb
))
442 udelay(100); /* allow for recovery */
445 case 3: /* not operational */
451 EXPORT_SYMBOL_GPL(cio_commit_config
);
454 * cio_update_schib - Perform stsch and update schib if subchannel is valid.
455 * @sch: subchannel on which to perform stsch
456 * Return zero on success, -ENODEV otherwise.
458 int cio_update_schib(struct subchannel
*sch
)
462 if (stsch(sch
->schid
, &schib
) || !css_sch_is_valid(&schib
))
465 memcpy(&sch
->schib
, &schib
, sizeof(schib
));
468 EXPORT_SYMBOL_GPL(cio_update_schib
);
471 * cio_enable_subchannel - enable a subchannel.
472 * @sch: subchannel to be enabled
473 * @intparm: interruption parameter to set
475 int cio_enable_subchannel(struct subchannel
*sch
, u32 intparm
)
479 CIO_TRACE_EVENT(2, "ensch");
480 CIO_TRACE_EVENT(2, dev_name(&sch
->dev
));
482 if (sch_is_pseudo_sch(sch
))
484 if (cio_update_schib(sch
))
488 sch
->config
.isc
= sch
->isc
;
489 sch
->config
.intparm
= intparm
;
491 ret
= cio_commit_config(sch
);
494 * Got a program check in msch. Try without
495 * the concurrent sense bit the next time.
497 sch
->config
.csense
= 0;
498 ret
= cio_commit_config(sch
);
500 CIO_HEX_EVENT(2, &ret
, sizeof(ret
));
503 EXPORT_SYMBOL_GPL(cio_enable_subchannel
);
506 * cio_disable_subchannel - disable a subchannel.
507 * @sch: subchannel to disable
509 int cio_disable_subchannel(struct subchannel
*sch
)
513 CIO_TRACE_EVENT(2, "dissch");
514 CIO_TRACE_EVENT(2, dev_name(&sch
->dev
));
516 if (sch_is_pseudo_sch(sch
))
518 if (cio_update_schib(sch
))
522 ret
= cio_commit_config(sch
);
524 CIO_HEX_EVENT(2, &ret
, sizeof(ret
));
527 EXPORT_SYMBOL_GPL(cio_disable_subchannel
);
530 * do_cio_interrupt() handles all normal I/O device IRQ's
532 static irqreturn_t
do_cio_interrupt(int irq
, void *dummy
)
534 struct tpi_info
*tpi_info
;
535 struct subchannel
*sch
;
538 set_cpu_flag(CIF_NOHZ_DELAY
);
539 tpi_info
= (struct tpi_info
*) &get_irq_regs()->int_code
;
540 trace_s390_cio_interrupt(tpi_info
);
541 irb
= this_cpu_ptr(&cio_irb
);
542 sch
= (struct subchannel
*)(unsigned long) tpi_info
->intparm
;
544 /* Clear pending interrupt condition. */
545 inc_irq_stat(IRQIO_CIO
);
546 tsch(tpi_info
->schid
, irb
);
549 spin_lock(sch
->lock
);
550 /* Store interrupt response block to lowcore. */
551 if (tsch(tpi_info
->schid
, irb
) == 0) {
552 /* Keep subchannel information word up to date. */
553 memcpy (&sch
->schib
.scsw
, &irb
->scsw
, sizeof (irb
->scsw
));
554 /* Call interrupt handler if there is one. */
555 if (sch
->driver
&& sch
->driver
->irq
)
556 sch
->driver
->irq(sch
);
558 inc_irq_stat(IRQIO_CIO
);
560 inc_irq_stat(IRQIO_CIO
);
561 spin_unlock(sch
->lock
);
566 static struct irqaction io_interrupt
= {
568 .handler
= do_cio_interrupt
,
571 void __init
init_cio_interrupts(void)
573 irq_set_chip_and_handler(IO_INTERRUPT
,
574 &dummy_irq_chip
, handle_percpu_irq
);
575 setup_irq(IO_INTERRUPT
, &io_interrupt
);
578 #ifdef CONFIG_CCW_CONSOLE
579 static struct subchannel
*console_sch
;
580 static struct lock_class_key console_sch_key
;
583 * Use cio_tsch to update the subchannel status and call the interrupt handler
584 * if status had been pending. Called with the subchannel's lock held.
586 void cio_tsch(struct subchannel
*sch
)
591 irb
= this_cpu_ptr(&cio_irb
);
592 /* Store interrupt response block to lowcore. */
593 if (tsch(sch
->schid
, irb
) != 0)
594 /* Not status pending or not operational. */
596 memcpy(&sch
->schib
.scsw
, &irb
->scsw
, sizeof(union scsw
));
597 /* Call interrupt handler with updated status. */
598 irq_context
= in_interrupt();
603 kstat_incr_irq_this_cpu(IO_INTERRUPT
);
604 if (sch
->driver
&& sch
->driver
->irq
)
605 sch
->driver
->irq(sch
);
607 inc_irq_stat(IRQIO_CIO
);
614 static int cio_test_for_console(struct subchannel_id schid
, void *data
)
618 if (stsch(schid
, &schib
) != 0)
620 if ((schib
.pmcw
.st
== SUBCHANNEL_TYPE_IO
) && schib
.pmcw
.dnv
&&
621 (schib
.pmcw
.dev
== console_devno
)) {
622 console_irq
= schid
.sch_no
;
623 return 1; /* found */
628 static int cio_get_console_sch_no(void)
630 struct subchannel_id schid
;
633 init_subchannel_id(&schid
);
634 if (console_irq
!= -1) {
635 /* VM provided us with the irq number of the console. */
636 schid
.sch_no
= console_irq
;
637 if (stsch(schid
, &schib
) != 0 ||
638 (schib
.pmcw
.st
!= SUBCHANNEL_TYPE_IO
) || !schib
.pmcw
.dnv
)
640 console_devno
= schib
.pmcw
.dev
;
641 } else if (console_devno
!= -1) {
642 /* At least the console device number is known. */
643 for_each_subchannel(cio_test_for_console
, NULL
);
648 struct subchannel
*cio_probe_console(void)
650 struct subchannel_id schid
;
651 struct subchannel
*sch
;
655 sch_no
= cio_get_console_sch_no();
657 pr_warn("No CCW console was found\n");
658 return ERR_PTR(-ENODEV
);
660 init_subchannel_id(&schid
);
661 schid
.sch_no
= sch_no
;
662 ret
= stsch(schid
, &schib
);
664 return ERR_PTR(-ENODEV
);
666 sch
= css_alloc_subchannel(schid
, &schib
);
670 lockdep_set_class(sch
->lock
, &console_sch_key
);
671 isc_register(CONSOLE_ISC
);
672 sch
->config
.isc
= CONSOLE_ISC
;
673 sch
->config
.intparm
= (u32
)(addr_t
)sch
;
674 ret
= cio_commit_config(sch
);
676 isc_unregister(CONSOLE_ISC
);
677 put_device(&sch
->dev
);
684 int cio_is_console(struct subchannel_id schid
)
688 return schid_equal(&schid
, &console_sch
->schid
);
691 void cio_register_early_subchannels(void)
698 ret
= css_register_subchannel(console_sch
);
700 put_device(&console_sch
->dev
);
702 #endif /* CONFIG_CCW_CONSOLE */
705 * cio_tm_start_key - perform start function
706 * @sch: subchannel on which to perform the start function
707 * @tcw: transport-command word to be started
708 * @lpm: mask of paths to use
709 * @key: storage key to use for storage access
711 * Start the tcw on the given subchannel. Return zero on success, non-zero
714 int cio_tm_start_key(struct subchannel
*sch
, struct tcw
*tcw
, u8 lpm
, u8 key
)
717 union orb
*orb
= &to_io_private(sch
)->orb
;
719 memset(orb
, 0, sizeof(union orb
));
720 orb
->tm
.intparm
= (u32
) (addr_t
) sch
;
721 orb
->tm
.key
= key
>> 4;
723 orb
->tm
.lpm
= lpm
? lpm
: sch
->lpm
;
724 orb
->tm
.tcw
= (u32
) (addr_t
) tcw
;
725 cc
= ssch(sch
->schid
, orb
);
733 return cio_start_handle_notoper(sch
, lpm
);
736 EXPORT_SYMBOL_GPL(cio_tm_start_key
);
739 * cio_tm_intrg - perform interrogate function
740 * @sch: subchannel on which to perform the interrogate function
742 * If the specified subchannel is running in transport-mode, perform the
743 * interrogate function. Return zero on success, non-zero otherwie.
745 int cio_tm_intrg(struct subchannel
*sch
)
749 if (!to_io_private(sch
)->orb
.tm
.b
)
751 cc
= xsch(sch
->schid
);
762 EXPORT_SYMBOL_GPL(cio_tm_intrg
);