kernel/trace/trace_preemptirq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * preemptoff and irqoff tracepoints
   4  *
   5  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
   6  */
   7
   8 #include <linux/kallsyms.h>
   9 #include <linux/uaccess.h>
  10 #include <linux/module.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/kprobes.h>
  13 #include "trace.h"
  14
  15 #define CREATE_TRACE_POINTS
  16 #include <trace/events/preemptirq.h>
  17
  18 #ifdef CONFIG_TRACE_IRQFLAGS
  19 /* Per-cpu variable to prevent redundant calls when IRQs already off */
  20 static DEFINE_PER_CPU(int, tracing_irq_cpu);
  21
  22 /*
  23  * Like trace_hardirqs_on() but without the lockdep invocation. This is
  24  * used in the low level entry code where the ordering vs. RCU is important
  25  * and lockdep uses a staged approach which splits the lockdep hardirq
  26  * tracking into a RCU on and a RCU off section.
  27  */
  28 void trace_hardirqs_on_prepare(void)
  29 {
  30         if (this_cpu_read(tracing_irq_cpu)) {
  31                 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
  32                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  33                 this_cpu_write(tracing_irq_cpu, 0);
  34         }
  35 }
  36 EXPORT_SYMBOL(trace_hardirqs_on_prepare);
  37 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
  38
  39 void trace_hardirqs_on(void)
  40 {
  41         if (this_cpu_read(tracing_irq_cpu)) {
  42                 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
  43                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
  44                 this_cpu_write(tracing_irq_cpu, 0);
  45         }
  46
  47         lockdep_hardirqs_on_prepare();
  48         lockdep_hardirqs_on(CALLER_ADDR0);
  49 }
  50 EXPORT_SYMBOL(trace_hardirqs_on);
  51 NOKPROBE_SYMBOL(trace_hardirqs_on);
  52
  53 /*
  54  * Like trace_hardirqs_off() but without the lockdep invocation. This is
  55  * used in the low level entry code where the ordering vs. RCU is important
  56  * and lockdep uses a staged approach which splits the lockdep hardirq
  57  * tracking into a RCU on and a RCU off section.
  58  */
  59 void trace_hardirqs_off_finish(void)
  60 {
  61         if (!this_cpu_read(tracing_irq_cpu)) {
  62                 this_cpu_write(tracing_irq_cpu, 1);
  63                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  64                 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
  65         }
  66
  67 }
  68 EXPORT_SYMBOL(trace_hardirqs_off_finish);
  69 NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
  70
  71 void trace_hardirqs_off(void)
  72 {
  73         lockdep_hardirqs_off(CALLER_ADDR0);
  74
  75         if (!this_cpu_read(tracing_irq_cpu)) {
  76                 this_cpu_write(tracing_irq_cpu, 1);
  77                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
  78                 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
  79         }
  80 }
  81 EXPORT_SYMBOL(trace_hardirqs_off);
  82 NOKPROBE_SYMBOL(trace_hardirqs_off);
  83 #endif /* CONFIG_TRACE_IRQFLAGS */
  84
  85 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
  86
  87 void trace_preempt_on(unsigned long a0, unsigned long a1)
  88 {
  89         trace_preempt_enable(a0, a1);
  90         tracer_preempt_on(a0, a1);
  91 }
  92
  93 void trace_preempt_off(unsigned long a0, unsigned long a1)
  94 {
  95         trace_preempt_disable(a0, a1);
  96         tracer_preempt_off(a0, a1);
  97 }
  98 #endif