1 // SPDX-License-Identifier: GPL-2.0
4 * Auto-group scheduling implementation:
7 unsigned int __read_mostly sysctl_sched_autogroup_enabled
= 1;
8 static struct autogroup autogroup_default
;
9 static atomic_t autogroup_seq_nr
;
12 static struct ctl_table sched_autogroup_sysctls
[] = {
14 .procname
= "sched_autogroup_enabled",
15 .data
= &sysctl_sched_autogroup_enabled
,
16 .maxlen
= sizeof(unsigned int),
18 .proc_handler
= proc_dointvec_minmax
,
19 .extra1
= SYSCTL_ZERO
,
24 static void __init
sched_autogroup_sysctl_init(void)
26 register_sysctl_init("kernel", sched_autogroup_sysctls
);
29 #define sched_autogroup_sysctl_init() do { } while (0)
32 void __init
autogroup_init(struct task_struct
*init_task
)
34 autogroup_default
.tg
= &root_task_group
;
35 kref_init(&autogroup_default
.kref
);
36 init_rwsem(&autogroup_default
.lock
);
37 init_task
->signal
->autogroup
= &autogroup_default
;
38 sched_autogroup_sysctl_init();
41 void autogroup_free(struct task_group
*tg
)
46 static inline void autogroup_destroy(struct kref
*kref
)
48 struct autogroup
*ag
= container_of(kref
, struct autogroup
, kref
);
50 #ifdef CONFIG_RT_GROUP_SCHED
51 /* We've redirected RT tasks to the root task group... */
55 sched_release_group(ag
->tg
);
56 sched_destroy_group(ag
->tg
);
59 static inline void autogroup_kref_put(struct autogroup
*ag
)
61 kref_put(&ag
->kref
, autogroup_destroy
);
64 static inline struct autogroup
*autogroup_kref_get(struct autogroup
*ag
)
70 static inline struct autogroup
*autogroup_task_get(struct task_struct
*p
)
75 if (!lock_task_sighand(p
, &flags
))
76 return autogroup_kref_get(&autogroup_default
);
78 ag
= autogroup_kref_get(p
->signal
->autogroup
);
79 unlock_task_sighand(p
, &flags
);
84 static inline struct autogroup
*autogroup_create(void)
86 struct autogroup
*ag
= kzalloc(sizeof(*ag
), GFP_KERNEL
);
87 struct task_group
*tg
;
92 tg
= sched_create_group(&root_task_group
);
97 init_rwsem(&ag
->lock
);
98 ag
->id
= atomic_inc_return(&autogroup_seq_nr
);
100 #ifdef CONFIG_RT_GROUP_SCHED
102 * Autogroup RT tasks are redirected to the root task group
103 * so we don't have to move tasks around upon policy change,
104 * or flail around trying to allocate bandwidth on the fly.
105 * A bandwidth exception in __sched_setscheduler() allows
106 * the policy change to proceed.
108 free_rt_sched_group(tg
);
109 tg
->rt_se
= root_task_group
.rt_se
;
110 tg
->rt_rq
= root_task_group
.rt_rq
;
114 sched_online_group(tg
, &root_task_group
);
120 if (printk_ratelimit()) {
121 printk(KERN_WARNING
"autogroup_create: %s failure.\n",
122 ag
? "sched_create_group()" : "kzalloc()");
125 return autogroup_kref_get(&autogroup_default
);
128 bool task_wants_autogroup(struct task_struct
*p
, struct task_group
*tg
)
130 if (tg
!= &root_task_group
)
133 * If we race with autogroup_move_group() the caller can use the old
134 * value of signal->autogroup but in this case sched_move_task() will
135 * be called again before autogroup_kref_put().
137 * However, there is no way sched_autogroup_exit_task() could tell us
138 * to avoid autogroup->tg, so we abuse PF_EXITING flag for this case.
140 if (p
->flags
& PF_EXITING
)
146 void sched_autogroup_exit_task(struct task_struct
*p
)
149 * We are going to call exit_notify() and autogroup_move_group() can't
150 * see this thread after that: we can no longer use signal->autogroup.
151 * See the PF_EXITING check in task_wants_autogroup().
157 autogroup_move_group(struct task_struct
*p
, struct autogroup
*ag
)
159 struct autogroup
*prev
;
160 struct task_struct
*t
;
163 if (WARN_ON_ONCE(!lock_task_sighand(p
, &flags
)))
166 prev
= p
->signal
->autogroup
;
168 unlock_task_sighand(p
, &flags
);
172 p
->signal
->autogroup
= autogroup_kref_get(ag
);
174 * We can't avoid sched_move_task() after we changed signal->autogroup,
175 * this process can already run with task_group() == prev->tg or we can
176 * race with cgroup code which can read autogroup = prev under rq->lock.
177 * In the latter case for_each_thread() can not miss a migrating thread,
178 * cpu_cgroup_attach() must not be possible after cgroup_exit() and it
179 * can't be removed from thread list, we hold ->siglock.
181 * If an exiting thread was already removed from thread list we rely on
182 * sched_autogroup_exit_task().
184 for_each_thread(p
, t
)
187 unlock_task_sighand(p
, &flags
);
188 autogroup_kref_put(prev
);
191 /* Allocates GFP_KERNEL, cannot be called under any spinlock: */
192 void sched_autogroup_create_attach(struct task_struct
*p
)
194 struct autogroup
*ag
= autogroup_create();
196 autogroup_move_group(p
, ag
);
198 /* Drop extra reference added by autogroup_create(): */
199 autogroup_kref_put(ag
);
201 EXPORT_SYMBOL(sched_autogroup_create_attach
);
203 /* Cannot be called under siglock. Currently has no users: */
204 void sched_autogroup_detach(struct task_struct
*p
)
206 autogroup_move_group(p
, &autogroup_default
);
208 EXPORT_SYMBOL(sched_autogroup_detach
);
210 void sched_autogroup_fork(struct signal_struct
*sig
)
212 sig
->autogroup
= autogroup_task_get(current
);
215 void sched_autogroup_exit(struct signal_struct
*sig
)
217 autogroup_kref_put(sig
->autogroup
);
220 static int __init
setup_autogroup(char *str
)
222 sysctl_sched_autogroup_enabled
= 0;
226 __setup("noautogroup", setup_autogroup
);
228 #ifdef CONFIG_PROC_FS
230 int proc_sched_autogroup_set_nice(struct task_struct
*p
, int nice
)
232 static unsigned long next
= INITIAL_JIFFIES
;
233 struct autogroup
*ag
;
234 unsigned long shares
;
237 if (nice
< MIN_NICE
|| nice
> MAX_NICE
)
240 err
= security_task_setnice(current
, nice
);
244 if (nice
< 0 && !can_nice(current
, nice
))
247 /* This is a heavy operation, taking global locks.. */
248 if (!capable(CAP_SYS_ADMIN
) && time_before(jiffies
, next
))
251 next
= HZ
/ 10 + jiffies
;
252 ag
= autogroup_task_get(p
);
254 idx
= array_index_nospec(nice
+ 20, 40);
255 shares
= scale_load(sched_prio_to_weight
[idx
]);
257 down_write(&ag
->lock
);
258 err
= sched_group_set_shares(ag
->tg
, shares
);
263 autogroup_kref_put(ag
);
268 void proc_sched_autogroup_show_task(struct task_struct
*p
, struct seq_file
*m
)
270 struct autogroup
*ag
= autogroup_task_get(p
);
272 if (!task_group_is_autogroup(ag
->tg
))
275 down_read(&ag
->lock
);
276 seq_printf(m
, "/autogroup-%ld nice %d\n", ag
->id
, ag
->nice
);
280 autogroup_kref_put(ag
);
282 #endif /* CONFIG_PROC_FS */
284 int autogroup_path(struct task_group
*tg
, char *buf
, int buflen
)
286 if (!task_group_is_autogroup(tg
))
289 return snprintf(buf
, buflen
, "%s-%ld", "/autogroup", tg
->autogroup
->id
);