arch/x86/include/asm/irq_stack.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ASM_X86_IRQ_STACK_H
   3 #define _ASM_X86_IRQ_STACK_H
   4
   5 #include <linux/ptrace.h>
   6
   7 #include <asm/processor.h>
   8
   9 #ifdef CONFIG_X86_64
  10 static __always_inline bool irqstack_active(void)
  11 {
  12         return __this_cpu_read(irq_count) != -1;
  13 }
  14
  15 void asm_call_on_stack(void *sp, void (*func)(void), void *arg);
  16 void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs),
  17                               struct pt_regs *regs);
  18 void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc),
  19                            struct irq_desc *desc);
  20
  21 static __always_inline void __run_on_irqstack(void (*func)(void))
  22 {
  23         void *tos = __this_cpu_read(hardirq_stack_ptr);
  24
  25         __this_cpu_add(irq_count, 1);
  26         asm_call_on_stack(tos - 8, func, NULL);
  27         __this_cpu_sub(irq_count, 1);
  28 }
  29
  30 static __always_inline void
  31 __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
  32                          struct pt_regs *regs)
  33 {
  34         void *tos = __this_cpu_read(hardirq_stack_ptr);
  35
  36         __this_cpu_add(irq_count, 1);
  37         asm_call_sysvec_on_stack(tos - 8, func, regs);
  38         __this_cpu_sub(irq_count, 1);
  39 }
  40
  41 static __always_inline void
  42 __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
  43                       struct irq_desc *desc)
  44 {
  45         void *tos = __this_cpu_read(hardirq_stack_ptr);
  46
  47         __this_cpu_add(irq_count, 1);
  48         asm_call_irq_on_stack(tos - 8, func, desc);
  49         __this_cpu_sub(irq_count, 1);
  50 }
  51
  52 #else /* CONFIG_X86_64 */
  53 static inline bool irqstack_active(void) { return false; }
  54 static inline void __run_on_irqstack(void (*func)(void)) { }
  55 static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs),
  56                                             struct pt_regs *regs) { }
  57 static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc),
  58                                          struct irq_desc *desc) { }
  59 #endif /* !CONFIG_X86_64 */
  60
  61 static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs)
  62 {
  63         if (IS_ENABLED(CONFIG_X86_32))
  64                 return false;
  65         if (!regs)
  66                 return !irqstack_active();
  67         return !user_mode(regs) && !irqstack_active();
  68 }
  69
  70
  71 static __always_inline void run_on_irqstack_cond(void (*func)(void),
  72                                                  struct pt_regs *regs)
  73 {
  74         lockdep_assert_irqs_disabled();
  75
  76         if (irq_needs_irq_stack(regs))
  77                 __run_on_irqstack(func);
  78         else
  79                 func();
  80 }
  81
  82 static __always_inline void
  83 run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs),
  84                             struct pt_regs *regs)
  85 {
  86         lockdep_assert_irqs_disabled();
  87
  88         if (irq_needs_irq_stack(regs))
  89                 __run_sysvec_on_irqstack(func, regs);
  90         else
  91                 func(regs);
  92 }
  93
  94 static __always_inline void
  95 run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc,
  96                          struct pt_regs *regs)
  97 {
  98         lockdep_assert_irqs_disabled();
  99
 100         if (irq_needs_irq_stack(regs))
 101                 __run_irq_on_irqstack(func, desc);
 102         else
 103                 func(desc);
 104 }
 105
 106 #endif