kernel/sched/isolation.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  Housekeeping management. Manage the targets for routine code that can run on
   4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
   5  *
   6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
   7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
   8  *
   9  */
  10
  11 enum hk_flags {
  12         HK_FLAG_TIMER           = BIT(HK_TYPE_TIMER),
  13         HK_FLAG_RCU             = BIT(HK_TYPE_RCU),
  14         HK_FLAG_MISC            = BIT(HK_TYPE_MISC),
  15         HK_FLAG_SCHED           = BIT(HK_TYPE_SCHED),
  16         HK_FLAG_TICK            = BIT(HK_TYPE_TICK),
  17         HK_FLAG_DOMAIN          = BIT(HK_TYPE_DOMAIN),
  18         HK_FLAG_WQ              = BIT(HK_TYPE_WQ),
  19         HK_FLAG_MANAGED_IRQ     = BIT(HK_TYPE_MANAGED_IRQ),
  20         HK_FLAG_KTHREAD         = BIT(HK_TYPE_KTHREAD),
  21 };
  22
  23 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  24 EXPORT_SYMBOL_GPL(housekeeping_overridden);
  25
  26 struct housekeeping {
  27         cpumask_var_t cpumasks[HK_TYPE_MAX];
  28         unsigned long flags;
  29 };
  30
  31 static struct housekeeping housekeeping;
  32
  33 bool housekeeping_enabled(enum hk_type type)
  34 {
  35         return !!(housekeeping.flags & BIT(type));
  36 }
  37 EXPORT_SYMBOL_GPL(housekeeping_enabled);
  38
  39 int housekeeping_any_cpu(enum hk_type type)
  40 {
  41         int cpu;
  42
  43         if (static_branch_unlikely(&housekeeping_overridden)) {
  44                 if (housekeeping.flags & BIT(type)) {
  45                         cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
  46                         if (cpu < nr_cpu_ids)
  47                                 return cpu;
  48
  49                         cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
  50                         if (likely(cpu < nr_cpu_ids))
  51                                 return cpu;
  52                         /*
  53                          * Unless we have another problem this can only happen
  54                          * at boot time before start_secondary() brings the 1st
  55                          * housekeeping CPU up.
  56                          */
  57                         WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
  58                                      type != HK_TYPE_TIMER);
  59                 }
  60         }
  61         return smp_processor_id();
  62 }
  63 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  64
  65 const struct cpumask *housekeeping_cpumask(enum hk_type type)
  66 {
  67         if (static_branch_unlikely(&housekeeping_overridden))
  68                 if (housekeeping.flags & BIT(type))
  69                         return housekeeping.cpumasks[type];
  70         return cpu_possible_mask;
  71 }
  72 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  73
  74 void housekeeping_affine(struct task_struct *t, enum hk_type type)
  75 {
  76         if (static_branch_unlikely(&housekeeping_overridden))
  77                 if (housekeeping.flags & BIT(type))
  78                         set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
  79 }
  80 EXPORT_SYMBOL_GPL(housekeeping_affine);
  81
  82 bool housekeeping_test_cpu(int cpu, enum hk_type type)
  83 {
  84         if (static_branch_unlikely(&housekeeping_overridden))
  85                 if (housekeeping.flags & BIT(type))
  86                         return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
  87         return true;
  88 }
  89 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  90
  91 void __init housekeeping_init(void)
  92 {
  93         enum hk_type type;
  94
  95         if (!housekeeping.flags)
  96                 return;
  97
  98         static_branch_enable(&housekeeping_overridden);
  99
 100         if (housekeeping.flags & HK_FLAG_TICK)
 101                 sched_tick_offload_init();
 102
 103         for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
 104                 /* We need at least one CPU to handle housekeeping work */
 105                 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
 106         }
 107 }
 108
 109 static void __init housekeeping_setup_type(enum hk_type type,
 110                                            cpumask_var_t housekeeping_staging)
 111 {
 112
 113         alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
 114         cpumask_copy(housekeeping.cpumasks[type],
 115                      housekeeping_staging);
 116 }
 117
 118 static int __init housekeeping_setup(char *str, unsigned long flags)
 119 {
 120         cpumask_var_t non_housekeeping_mask, housekeeping_staging;
 121         unsigned int first_cpu;
 122         int err = 0;
 123
 124         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
 125                 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
 126                         pr_warn("Housekeeping: nohz unsupported."
 127                                 " Build with CONFIG_NO_HZ_FULL\n");
 128                         return 0;
 129                 }
 130         }
 131
 132         alloc_bootmem_cpumask_var(&non_housekeeping_mask);
 133         if (cpulist_parse(str, non_housekeeping_mask) < 0) {
 134                 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
 135                 goto free_non_housekeeping_mask;
 136         }
 137
 138         alloc_bootmem_cpumask_var(&housekeeping_staging);
 139         cpumask_andnot(housekeeping_staging,
 140                        cpu_possible_mask, non_housekeeping_mask);
 141
 142         first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
 143         if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
 144                 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
 145                 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
 146                 if (!housekeeping.flags) {
 147                         pr_warn("Housekeeping: must include one present CPU, "
 148                                 "using boot CPU:%d\n", smp_processor_id());
 149                 }
 150         }
 151
 152         if (cpumask_empty(non_housekeeping_mask))
 153                 goto free_housekeeping_staging;
 154
 155         if (!housekeeping.flags) {
 156                 /* First setup call ("nohz_full=" or "isolcpus=") */
 157                 enum hk_type type;
 158
 159                 for_each_set_bit(type, &flags, HK_TYPE_MAX)
 160                         housekeeping_setup_type(type, housekeeping_staging);
 161         } else {
 162                 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
 163                 enum hk_type type;
 164                 unsigned long iter_flags = flags & housekeeping.flags;
 165
 166                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
 167                         if (!cpumask_equal(housekeeping_staging,
 168                                            housekeeping.cpumasks[type])) {
 169                                 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
 170                                 goto free_housekeeping_staging;
 171                         }
 172                 }
 173
 174                 iter_flags = flags & ~housekeeping.flags;
 175
 176                 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
 177                         housekeeping_setup_type(type, housekeeping_staging);
 178         }
 179
 180         if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
 181                 tick_nohz_full_setup(non_housekeeping_mask);
 182
 183         housekeeping.flags |= flags;
 184         err = 1;
 185
 186 free_housekeeping_staging:
 187         free_bootmem_cpumask_var(housekeeping_staging);
 188 free_non_housekeeping_mask:
 189         free_bootmem_cpumask_var(non_housekeeping_mask);
 190
 191         return err;
 192 }
 193
 194 static int __init housekeeping_nohz_full_setup(char *str)
 195 {
 196         unsigned long flags;
 197
 198         flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
 199                 HK_FLAG_MISC | HK_FLAG_KTHREAD;
 200
 201         return housekeeping_setup(str, flags);
 202 }
 203 __setup("nohz_full=", housekeeping_nohz_full_setup);
 204
 205 static int __init housekeeping_isolcpus_setup(char *str)
 206 {
 207         unsigned long flags = 0;
 208         bool illegal = false;
 209         char *par;
 210         int len;
 211
 212         while (isalpha(*str)) {
 213                 if (!strncmp(str, "nohz,", 5)) {
 214                         str += 5;
 215                         flags |= HK_FLAG_TICK;
 216                         continue;
 217                 }
 218
 219                 if (!strncmp(str, "domain,", 7)) {
 220                         str += 7;
 221                         flags |= HK_FLAG_DOMAIN;
 222                         continue;
 223                 }
 224
 225                 if (!strncmp(str, "managed_irq,", 12)) {
 226                         str += 12;
 227                         flags |= HK_FLAG_MANAGED_IRQ;
 228                         continue;
 229                 }
 230
 231                 /*
 232                  * Skip unknown sub-parameter and validate that it is not
 233                  * containing an invalid character.
 234                  */
 235                 for (par = str, len = 0; *str && *str != ','; str++, len++) {
 236                         if (!isalpha(*str) && *str != '_')
 237                                 illegal = true;
 238                 }
 239
 240                 if (illegal) {
 241                         pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
 242                         return 0;
 243                 }
 244
 245                 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
 246                 str++;
 247         }
 248
 249         /* Default behaviour for isolcpus without flags */
 250         if (!flags)
 251                 flags |= HK_FLAG_DOMAIN;
 252
 253         return housekeeping_setup(str, flags);
 254 }
 255 __setup("isolcpus=", housekeeping_isolcpus_setup);