kernel/sched/isolation.c

   1 /*
   2  *  Housekeeping management. Manage the targets for routine code that can run on
   3  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   4  *
   5  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   6  *
   7  */
   8
   9 #include <linux/sched/isolation.h>
  10 #include <linux/tick.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/static_key.h>
  14 #include <linux/ctype.h>
  15
  16 DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
  17 EXPORT_SYMBOL_GPL(housekeeping_overriden);
  18 static cpumask_var_t housekeeping_mask;
  19 static unsigned int housekeeping_flags;
  20
  21 int housekeeping_any_cpu(enum hk_flags flags)
  22 {
  23         if (static_branch_unlikely(&housekeeping_overriden))
  24                 if (housekeeping_flags & flags)
  25                         return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  26         return smp_processor_id();
  27 }
  28 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  29
  30 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  31 {
  32         if (static_branch_unlikely(&housekeeping_overriden))
  33                 if (housekeeping_flags & flags)
  34                         return housekeeping_mask;
  35         return cpu_possible_mask;
  36 }
  37 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  38
  39 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  40 {
  41         if (static_branch_unlikely(&housekeeping_overriden))
  42                 if (housekeeping_flags & flags)
  43                         set_cpus_allowed_ptr(t, housekeeping_mask);
  44 }
  45 EXPORT_SYMBOL_GPL(housekeeping_affine);
  46
  47 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  48 {
  49         if (static_branch_unlikely(&housekeeping_overriden))
  50                 if (housekeeping_flags & flags)
  51                         return cpumask_test_cpu(cpu, housekeeping_mask);
  52         return true;
  53 }
  54 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  55
  56 void __init housekeeping_init(void)
  57 {
  58         if (!housekeeping_flags)
  59                 return;
  60
  61         static_branch_enable(&housekeeping_overriden);
  62
  63         /* We need at least one CPU to handle housekeeping work */
  64         WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  65 }
  66
  67 static int __init housekeeping_setup(char *str, enum hk_flags flags)
  68 {
  69         cpumask_var_t non_housekeeping_mask;
  70         int err;
  71
  72         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  73         err = cpulist_parse(str, non_housekeeping_mask);
  74         if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  75                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  76                 free_bootmem_cpumask_var(non_housekeeping_mask);
  77                 return 0;
  78         }
  79
  80         if (!housekeeping_flags) {
  81                 alloc_bootmem_cpumask_var(&housekeeping_mask);
  82                 cpumask_andnot(housekeeping_mask,
  83                                cpu_possible_mask, non_housekeeping_mask);
  84                 if (cpumask_empty(housekeeping_mask))
  85                         cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  86         } else {
  87                 cpumask_var_t tmp;
  88
  89                 alloc_bootmem_cpumask_var(&tmp);
  90                 cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  91                 if (!cpumask_equal(tmp, housekeeping_mask)) {
  92                         pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  93                         free_bootmem_cpumask_var(tmp);
  94                         free_bootmem_cpumask_var(non_housekeeping_mask);
  95                         return 0;
  96                 }
  97                 free_bootmem_cpumask_var(tmp);
  98         }
  99
 100         if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
 101                 if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 102                         tick_nohz_full_setup(non_housekeeping_mask);
 103                 } else {
 104                         pr_warn("Housekeeping: nohz unsupported."
 105                                 " Build with CONFIG_NO_HZ_FULL\n");
 106                         free_bootmem_cpumask_var(non_housekeeping_mask);
 107                         return 0;
 108                 }
 109         }
 110
 111         housekeeping_flags |= flags;
 112
 113         free_bootmem_cpumask_var(non_housekeeping_mask);
 114
 115         return 1;
 116 }
 117
 118 static int __init housekeeping_nohz_full_setup(char *str)
 119 {
 120         unsigned int flags;
 121
 122         flags = HK_FLAG_TICK | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC;
 123
 124         return housekeeping_setup(str, flags);
 125 }
 126 __setup("nohz_full=", housekeeping_nohz_full_setup);
 127
 128 static int __init housekeeping_isolcpus_setup(char *str)
 129 {
 130         unsigned int flags = 0;
 131
 132         while (isalpha(*str)) {
 133                 if (!strncmp(str, "nohz,", 5)) {
 134                         str += 5;
 135                         flags |= HK_FLAG_TICK;
 136                         continue;
 137                 }
 138
 139                 if (!strncmp(str, "domain,", 7)) {
 140                         str += 7;
 141                         flags |= HK_FLAG_DOMAIN;
 142                         continue;
 143                 }
 144
 145                 pr_warn("isolcpus: Error, unknown flag\n");
 146                 return 0;
 147         }
 148
 149         /* Default behaviour for isolcpus without flags */
 150         if (!flags)
 151                 flags |= HK_FLAG_DOMAIN;
 152
 153         return housekeeping_setup(str, flags);
 154 }
 155 __setup("isolcpus=", housekeeping_isolcpus_setup);