mm/damon/lru_sort.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DAMON-based LRU-lists Sorting
   4  *
   5  * Author: SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #define pr_fmt(fmt) "damon-lru-sort: " fmt
   9
  10 #include <linux/damon.h>
  11 #include <linux/kstrtox.h>
  12 #include <linux/module.h>
  13
  14 #include "modules-common.h"
  15
  16 #ifdef MODULE_PARAM_PREFIX
  17 #undef MODULE_PARAM_PREFIX
  18 #endif
  19 #define MODULE_PARAM_PREFIX "damon_lru_sort."
  20
  21 /*
  22  * Enable or disable DAMON_LRU_SORT.
  23  *
  24  * You can enable DAMON_LRU_SORT by setting the value of this parameter as
  25  * ``Y``.  Setting it as ``N`` disables DAMON_LRU_SORT.  Note that
  26  * DAMON_LRU_SORT could do no real monitoring and LRU-lists sorting due to the
  27  * watermarks-based activation condition.  Refer to below descriptions for the
  28  * watermarks parameter for this.
  29  */
  30 static bool enabled __read_mostly;
  31
  32 /*
  33  * Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``.
  34  *
  35  * Input parameters that updated while DAMON_LRU_SORT is running are not
  36  * applied by default.  Once this parameter is set as ``Y``, DAMON_LRU_SORT
  37  * reads values of parametrs except ``enabled`` again.  Once the re-reading is
  38  * done, this parameter is set as ``N``.  If invalid parameters are found while
  39  * the re-reading, DAMON_LRU_SORT will be disabled.
  40  */
  41 static bool commit_inputs __read_mostly;
  42 module_param(commit_inputs, bool, 0600);
  43
  44 /*
  45  * Access frequency threshold for hot memory regions identification in permil.
  46  *
  47  * If a memory region is accessed in frequency of this or higher,
  48  * DAMON_LRU_SORT identifies the region as hot, and mark it as accessed on the
  49  * LRU list, so that it could not be reclaimed under memory pressure.  50% by
  50  * default.
  51  */
  52 static unsigned long hot_thres_access_freq = 500;
  53 module_param(hot_thres_access_freq, ulong, 0600);
  54
  55 /*
  56  * Time threshold for cold memory regions identification in microseconds.
  57  *
  58  * If a memory region is not accessed for this or longer time, DAMON_LRU_SORT
  59  * identifies the region as cold, and mark it as unaccessed on the LRU list, so
  60  * that it could be reclaimed first under memory pressure.  120 seconds by
  61  * default.
  62  */
  63 static unsigned long cold_min_age __read_mostly = 120000000;
  64 module_param(cold_min_age, ulong, 0600);
  65
  66 static struct damos_quota damon_lru_sort_quota = {
  67         /* Use up to 10 ms per 1 sec, by default */
  68         .ms = 10,
  69         .sz = 0,
  70         .reset_interval = 1000,
  71         /* Within the quota, mark hotter regions accessed first. */
  72         .weight_sz = 0,
  73         .weight_nr_accesses = 1,
  74         .weight_age = 0,
  75 };
  76 DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota);
  77
  78 static struct damos_watermarks damon_lru_sort_wmarks = {
  79         .metric = DAMOS_WMARK_FREE_MEM_RATE,
  80         .interval = 5000000,    /* 5 seconds */
  81         .high = 200,            /* 20 percent */
  82         .mid = 150,             /* 15 percent */
  83         .low = 50,              /* 5 percent */
  84 };
  85 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks);
  86
  87 static struct damon_attrs damon_lru_sort_mon_attrs = {
  88         .sample_interval = 5000,        /* 5 ms */
  89         .aggr_interval = 100000,        /* 100 ms */
  90         .ops_update_interval = 0,
  91         .min_nr_regions = 10,
  92         .max_nr_regions = 1000,
  93 };
  94 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs);
  95
  96 /*
  97  * Start of the target memory region in physical address.
  98  *
  99  * The start physical address of memory region that DAMON_LRU_SORT will do work
 100  * against.  By default, biggest System RAM is used as the region.
 101  */
 102 static unsigned long monitor_region_start __read_mostly;
 103 module_param(monitor_region_start, ulong, 0600);
 104
 105 /*
 106  * End of the target memory region in physical address.
 107  *
 108  * The end physical address of memory region that DAMON_LRU_SORT will do work
 109  * against.  By default, biggest System RAM is used as the region.
 110  */
 111 static unsigned long monitor_region_end __read_mostly;
 112 module_param(monitor_region_end, ulong, 0600);
 113
 114 /*
 115  * PID of the DAMON thread
 116  *
 117  * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread.
 118  * Else, -1.
 119  */
 120 static int kdamond_pid __read_mostly = -1;
 121 module_param(kdamond_pid, int, 0400);
 122
 123 static struct damos_stat damon_lru_sort_hot_stat;
 124 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat,
 125                 lru_sort_tried_hot_regions, lru_sorted_hot_regions,
 126                 hot_quota_exceeds);
 127
 128 static struct damos_stat damon_lru_sort_cold_stat;
 129 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat,
 130                 lru_sort_tried_cold_regions, lru_sorted_cold_regions,
 131                 cold_quota_exceeds);
 132
 133 static struct damos_access_pattern damon_lru_sort_stub_pattern = {
 134         /* Find regions having PAGE_SIZE or larger size */
 135         .min_sz_region = PAGE_SIZE,
 136         .max_sz_region = ULONG_MAX,
 137         /* no matter its access frequency */
 138         .min_nr_accesses = 0,
 139         .max_nr_accesses = UINT_MAX,
 140         /* no matter its age */
 141         .min_age_region = 0,
 142         .max_age_region = UINT_MAX,
 143 };
 144
 145 static struct damon_ctx *ctx;
 146 static struct damon_target *target;
 147
 148 static struct damos *damon_lru_sort_new_scheme(
 149                 struct damos_access_pattern *pattern, enum damos_action action)
 150 {
 151         struct damos_quota quota = damon_lru_sort_quota;
 152
 153         /* Use half of total quota for hot/cold pages sorting */
 154         quota.ms = quota.ms / 2;
 155
 156         return damon_new_scheme(
 157                         /* find the pattern, and */
 158                         pattern,
 159                         /* (de)prioritize on LRU-lists */
 160                         action,
 161                         /* for each aggregation interval */
 162                         0,
 163                         /* under the quota. */
 164                         &quota,
 165                         /* (De)activate this according to the watermarks. */
 166                         &damon_lru_sort_wmarks,
 167                         NUMA_NO_NODE);
 168 }
 169
 170 /* Create a DAMON-based operation scheme for hot memory regions */
 171 static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres)
 172 {
 173         struct damos_access_pattern pattern = damon_lru_sort_stub_pattern;
 174
 175         pattern.min_nr_accesses = hot_thres;
 176         return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO);
 177 }
 178
 179 /* Create a DAMON-based operation scheme for cold memory regions */
 180 static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres)
 181 {
 182         struct damos_access_pattern pattern = damon_lru_sort_stub_pattern;
 183
 184         pattern.max_nr_accesses = 0;
 185         pattern.min_age_region = cold_thres;
 186         return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO);
 187 }
 188
 189 static int damon_lru_sort_apply_parameters(void)
 190 {
 191         struct damon_ctx *param_ctx;
 192         struct damon_target *param_target;
 193         struct damos *hot_scheme, *cold_scheme;
 194         unsigned int hot_thres, cold_thres;
 195         int err;
 196
 197         err = damon_modules_new_paddr_ctx_target(&param_ctx, &param_target);
 198         if (err)
 199                 return err;
 200
 201         err = damon_set_attrs(ctx, &damon_lru_sort_mon_attrs);
 202         if (err)
 203                 goto out;
 204
 205         err = -ENOMEM;
 206         hot_thres = damon_max_nr_accesses(&damon_lru_sort_mon_attrs) *
 207                 hot_thres_access_freq / 1000;
 208         hot_scheme = damon_lru_sort_new_hot_scheme(hot_thres);
 209         if (!hot_scheme)
 210                 goto out;
 211
 212         cold_thres = cold_min_age / damon_lru_sort_mon_attrs.aggr_interval;
 213         cold_scheme = damon_lru_sort_new_cold_scheme(cold_thres);
 214         if (!cold_scheme) {
 215                 damon_destroy_scheme(hot_scheme);
 216                 goto out;
 217         }
 218
 219         damon_set_schemes(param_ctx, &hot_scheme, 1);
 220         damon_add_scheme(param_ctx, cold_scheme);
 221
 222         err = damon_set_region_biggest_system_ram_default(param_target,
 223                                         &monitor_region_start,
 224                                         &monitor_region_end);
 225         if (err)
 226                 goto out;
 227         err = damon_commit_ctx(ctx, param_ctx);
 228 out:
 229         damon_destroy_ctx(param_ctx);
 230         return err;
 231 }
 232
 233 static int damon_lru_sort_turn(bool on)
 234 {
 235         int err;
 236
 237         if (!on) {
 238                 err = damon_stop(&ctx, 1);
 239                 if (!err)
 240                         kdamond_pid = -1;
 241                 return err;
 242         }
 243
 244         err = damon_lru_sort_apply_parameters();
 245         if (err)
 246                 return err;
 247
 248         err = damon_start(&ctx, 1, true);
 249         if (err)
 250                 return err;
 251         kdamond_pid = ctx->kdamond->pid;
 252         return 0;
 253 }
 254
 255 static int damon_lru_sort_enabled_store(const char *val,
 256                 const struct kernel_param *kp)
 257 {
 258         bool is_enabled = enabled;
 259         bool enable;
 260         int err;
 261
 262         err = kstrtobool(val, &enable);
 263         if (err)
 264                 return err;
 265
 266         if (is_enabled == enable)
 267                 return 0;
 268
 269         /* Called before init function.  The function will handle this. */
 270         if (!ctx)
 271                 goto set_param_out;
 272
 273         err = damon_lru_sort_turn(enable);
 274         if (err)
 275                 return err;
 276
 277 set_param_out:
 278         enabled = enable;
 279         return err;
 280 }
 281
 282 static const struct kernel_param_ops enabled_param_ops = {
 283         .set = damon_lru_sort_enabled_store,
 284         .get = param_get_bool,
 285 };
 286
 287 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 288 MODULE_PARM_DESC(enabled,
 289         "Enable or disable DAMON_LRU_SORT (default: disabled)");
 290
 291 static int damon_lru_sort_handle_commit_inputs(void)
 292 {
 293         int err;
 294
 295         if (!commit_inputs)
 296                 return 0;
 297
 298         err = damon_lru_sort_apply_parameters();
 299         commit_inputs = false;
 300         return err;
 301 }
 302
 303 static int damon_lru_sort_after_aggregation(struct damon_ctx *c)
 304 {
 305         struct damos *s;
 306
 307         /* update the stats parameter */
 308         damon_for_each_scheme(s, c) {
 309                 if (s->action == DAMOS_LRU_PRIO)
 310                         damon_lru_sort_hot_stat = s->stat;
 311                 else if (s->action == DAMOS_LRU_DEPRIO)
 312                         damon_lru_sort_cold_stat = s->stat;
 313         }
 314
 315         return damon_lru_sort_handle_commit_inputs();
 316 }
 317
 318 static int damon_lru_sort_after_wmarks_check(struct damon_ctx *c)
 319 {
 320         return damon_lru_sort_handle_commit_inputs();
 321 }
 322
 323 static int __init damon_lru_sort_init(void)
 324 {
 325         int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
 326
 327         if (err)
 328                 return err;
 329
 330         ctx->callback.after_wmarks_check = damon_lru_sort_after_wmarks_check;
 331         ctx->callback.after_aggregation = damon_lru_sort_after_aggregation;
 332
 333         /* 'enabled' has set before this function, probably via command line */
 334         if (enabled)
 335                 err = damon_lru_sort_turn(true);
 336
 337         return err;
 338 }
 339
 340 module_init(damon_lru_sort_init);