block/blk-ioprio.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Block rq-qos policy for assigning an I/O priority class to requests.
   4  *
   5  * Using an rq-qos policy for assigning I/O priority class has two advantages
   6  * over using the ioprio_set() system call:
   7  *
   8  * - This policy is cgroup based so it has all the advantages of cgroups.
   9  * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos
  10  *   controller affects page cache writeback I/O for filesystems that support
  11  *   assiociating a cgroup with writeback I/O. See also
  12  *   Documentation/admin-guide/cgroup-v2.rst.
  13  */
  14
  15 #include <linux/blk-mq.h>
  16 #include <linux/blk_types.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include "blk-cgroup.h"
  20 #include "blk-ioprio.h"
  21 #include "blk-rq-qos.h"
  22
  23 /**
  24  * enum prio_policy - I/O priority class policy.
  25  * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class.
  26  * @POLICY_PROMOTE_TO_RT: modify no-IOPRIO_CLASS_RT to IOPRIO_CLASS_RT.
  27  * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into
  28  *              IOPRIO_CLASS_BE.
  29  * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE.
  30  * @POLICY_NONE_TO_RT: an alias for POLICY_PROMOTE_TO_RT.
  31  *
  32  * See also <linux/ioprio.h>.
  33  */
  34 enum prio_policy {
  35         POLICY_NO_CHANGE        = 0,
  36         POLICY_PROMOTE_TO_RT    = 1,
  37         POLICY_RESTRICT_TO_BE   = 2,
  38         POLICY_ALL_TO_IDLE      = 3,
  39         POLICY_NONE_TO_RT       = 4,
  40 };
  41
  42 static const char *policy_name[] = {
  43         [POLICY_NO_CHANGE]      = "no-change",
  44         [POLICY_PROMOTE_TO_RT]  = "promote-to-rt",
  45         [POLICY_RESTRICT_TO_BE] = "restrict-to-be",
  46         [POLICY_ALL_TO_IDLE]    = "idle",
  47         [POLICY_NONE_TO_RT]     = "none-to-rt",
  48 };
  49
  50 static struct blkcg_policy ioprio_policy;
  51
  52 /**
  53  * struct ioprio_blkcg - Per cgroup data.
  54  * @cpd: blkcg_policy_data structure.
  55  * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>.
  56  */
  57 struct ioprio_blkcg {
  58         struct blkcg_policy_data cpd;
  59         enum prio_policy         prio_policy;
  60 };
  61
  62 static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
  63 {
  64         return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
  65                             struct ioprio_blkcg, cpd);
  66 }
  67
  68 static struct ioprio_blkcg *
  69 ioprio_blkcg_from_css(struct cgroup_subsys_state *css)
  70 {
  71         return blkcg_to_ioprio_blkcg(css_to_blkcg(css));
  72 }
  73
  74 static int ioprio_show_prio_policy(struct seq_file *sf, void *v)
  75 {
  76         struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
  77
  78         seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
  79         return 0;
  80 }
  81
  82 static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf,
  83                                       size_t nbytes, loff_t off)
  84 {
  85         struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
  86         int ret;
  87
  88         if (off != 0)
  89                 return -EIO;
  90         /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */
  91         ret = sysfs_match_string(policy_name, buf);
  92         if (ret < 0)
  93                 return ret;
  94         blkcg->prio_policy = ret;
  95         return nbytes;
  96 }
  97
  98 static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp)
  99 {
 100         struct ioprio_blkcg *blkcg;
 101
 102         blkcg = kzalloc(sizeof(*blkcg), gfp);
 103         if (!blkcg)
 104                 return NULL;
 105         blkcg->prio_policy = POLICY_NO_CHANGE;
 106         return &blkcg->cpd;
 107 }
 108
 109 static void ioprio_free_cpd(struct blkcg_policy_data *cpd)
 110 {
 111         struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
 112
 113         kfree(blkcg);
 114 }
 115
 116 #define IOPRIO_ATTRS                                            \
 117         {                                                       \
 118                 .name           = "prio.class",                 \
 119                 .seq_show       = ioprio_show_prio_policy,      \
 120                 .write          = ioprio_set_prio_policy,       \
 121         },                                                      \
 122         { } /* sentinel */
 123
 124 /* cgroup v2 attributes */
 125 static struct cftype ioprio_files[] = {
 126         IOPRIO_ATTRS
 127 };
 128
 129 /* cgroup v1 attributes */
 130 static struct cftype ioprio_legacy_files[] = {
 131         IOPRIO_ATTRS
 132 };
 133
 134 static struct blkcg_policy ioprio_policy = {
 135         .dfl_cftypes    = ioprio_files,
 136         .legacy_cftypes = ioprio_legacy_files,
 137
 138         .cpd_alloc_fn   = ioprio_alloc_cpd,
 139         .cpd_free_fn    = ioprio_free_cpd,
 140 };
 141
 142 void blkcg_set_ioprio(struct bio *bio)
 143 {
 144         struct ioprio_blkcg *blkcg = blkcg_to_ioprio_blkcg(bio->bi_blkg->blkcg);
 145         u16 prio;
 146
 147         if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE)
 148                 return;
 149
 150         if (blkcg->prio_policy == POLICY_PROMOTE_TO_RT ||
 151             blkcg->prio_policy == POLICY_NONE_TO_RT) {
 152                 /*
 153                  * For RT threads, the default priority level is 4 because
 154                  * task_nice is 0. By promoting non-RT io-priority to RT-class
 155                  * and default level 4, those requests that are already
 156                  * RT-class but need a higher io-priority can use ioprio_set()
 157                  * to achieve this.
 158                  */
 159                 if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) != IOPRIO_CLASS_RT)
 160                         bio->bi_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 4);
 161                 return;
 162         }
 163
 164         /*
 165          * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
 166          * correspond to a lower priority. Hence, the max_t() below selects
 167          * the lower priority of bi_ioprio and the cgroup I/O priority class.
 168          * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O
 169          * priority is assigned to the bio.
 170          */
 171         prio = max_t(u16, bio->bi_ioprio,
 172                         IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
 173         if (prio > bio->bi_ioprio)
 174                 bio->bi_ioprio = prio;
 175 }
 176
 177 static int __init ioprio_init(void)
 178 {
 179         return blkcg_policy_register(&ioprio_policy);
 180 }
 181
 182 static void __exit ioprio_exit(void)
 183 {
 184         blkcg_policy_unregister(&ioprio_policy);
 185 }
 186
 187 module_init(ioprio_init);
 188 module_exit(ioprio_exit);