mm/damon/reclaim.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DAMON-based page reclamation
   4  *
   5  * Author: SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #define pr_fmt(fmt) "damon-reclaim: " fmt
   9
  10 #include <linux/damon.h>
  11 #include <linux/kstrtox.h>
  12 #include <linux/module.h>
  13
  14 #include "modules-common.h"
  15
  16 #ifdef MODULE_PARAM_PREFIX
  17 #undef MODULE_PARAM_PREFIX
  18 #endif
  19 #define MODULE_PARAM_PREFIX "damon_reclaim."
  20
  21 /*
  22  * Enable or disable DAMON_RECLAIM.
  23  *
  24  * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
  25  * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
  26  * do no real monitoring and reclamation due to the watermarks-based activation
  27  * condition.  Refer to below descriptions for the watermarks parameter for
  28  * this.
  29  */
  30 static bool enabled __read_mostly;
  31
  32 /*
  33  * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
  34  *
  35  * Input parameters that updated while DAMON_RECLAIM is running are not applied
  36  * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
  37  * of parametrs except ``enabled`` again.  Once the re-reading is done, this
  38  * parameter is set as ``N``.  If invalid parameters are found while the
  39  * re-reading, DAMON_RECLAIM will be disabled.
  40  */
  41 static bool commit_inputs __read_mostly;
  42 module_param(commit_inputs, bool, 0600);
  43
  44 /*
  45  * Time threshold for cold memory regions identification in microseconds.
  46  *
  47  * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
  48  * identifies the region as cold, and reclaims.  120 seconds by default.
  49  */
  50 static unsigned long min_age __read_mostly = 120000000;
  51 module_param(min_age, ulong, 0600);
  52
  53 static struct damos_quota damon_reclaim_quota = {
  54         /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
  55         .ms = 10,
  56         .sz = 128 * 1024 * 1024,
  57         .reset_interval = 1000,
  58         /* Within the quota, page out older regions first. */
  59         .weight_sz = 0,
  60         .weight_nr_accesses = 0,
  61         .weight_age = 1
  62 };
  63 DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
  64
  65 /*
  66  * Desired level of memory pressure-stall time in microseconds.
  67  *
  68  * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
  69  * increases and decreases the effective level of the quota aiming this level of
  70  * memory pressure is incurred.  System-wide ``some`` memory PSI in microseconds
  71  * per quota reset interval (``quota_reset_interval_ms``) is collected and
  72  * compared to this value to see if the aim is satisfied.  Value zero means
  73  * disabling this auto-tuning feature.
  74  *
  75  * Disabled by default.
  76  */
  77 static unsigned long quota_mem_pressure_us __read_mostly;
  78 module_param(quota_mem_pressure_us, ulong, 0600);
  79
  80 /*
  81  * User-specifiable feedback for auto-tuning of the effective quota.
  82  *
  83  * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
  84  * increases and decreases the effective level of the quota aiming receiving this
  85  * feedback of value ``10,000`` from the user.  DAMON_RECLAIM assumes the feedback
  86  * value and the quota are positively proportional.  Value zero means disabling
  87  * this auto-tuning feature.
  88  *
  89  * Disabled by default.
  90  *
  91  */
  92 static unsigned long quota_autotune_feedback __read_mostly;
  93 module_param(quota_autotune_feedback, ulong, 0600);
  94
  95 static struct damos_watermarks damon_reclaim_wmarks = {
  96         .metric = DAMOS_WMARK_FREE_MEM_RATE,
  97         .interval = 5000000,    /* 5 seconds */
  98         .high = 500,            /* 50 percent */
  99         .mid = 400,             /* 40 percent */
 100         .low = 200,             /* 20 percent */
 101 };
 102 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
 103
 104 static struct damon_attrs damon_reclaim_mon_attrs = {
 105         .sample_interval = 5000,        /* 5 ms */
 106         .aggr_interval = 100000,        /* 100 ms */
 107         .ops_update_interval = 0,
 108         .min_nr_regions = 10,
 109         .max_nr_regions = 1000,
 110 };
 111 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
 112
 113 /*
 114  * Start of the target memory region in physical address.
 115  *
 116  * The start physical address of memory region that DAMON_RECLAIM will do work
 117  * against.  By default, biggest System RAM is used as the region.
 118  */
 119 static unsigned long monitor_region_start __read_mostly;
 120 module_param(monitor_region_start, ulong, 0600);
 121
 122 /*
 123  * End of the target memory region in physical address.
 124  *
 125  * The end physical address of memory region that DAMON_RECLAIM will do work
 126  * against.  By default, biggest System RAM is used as the region.
 127  */
 128 static unsigned long monitor_region_end __read_mostly;
 129 module_param(monitor_region_end, ulong, 0600);
 130
 131 /*
 132  * Skip anonymous pages reclamation.
 133  *
 134  * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous
 135  * pages.  By default, ``N``.
 136  */
 137 static bool skip_anon __read_mostly;
 138 module_param(skip_anon, bool, 0600);
 139
 140 /*
 141  * PID of the DAMON thread
 142  *
 143  * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
 144  * Else, -1.
 145  */
 146 static int kdamond_pid __read_mostly = -1;
 147 module_param(kdamond_pid, int, 0400);
 148
 149 static struct damos_stat damon_reclaim_stat;
 150 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
 151                 reclaim_tried_regions, reclaimed_regions, quota_exceeds);
 152
 153 static struct damon_ctx *ctx;
 154 static struct damon_target *target;
 155
 156 static struct damos *damon_reclaim_new_scheme(void)
 157 {
 158         struct damos_access_pattern pattern = {
 159                 /* Find regions having PAGE_SIZE or larger size */
 160                 .min_sz_region = PAGE_SIZE,
 161                 .max_sz_region = ULONG_MAX,
 162                 /* and not accessed at all */
 163                 .min_nr_accesses = 0,
 164                 .max_nr_accesses = 0,
 165                 /* for min_age or more micro-seconds */
 166                 .min_age_region = min_age /
 167                         damon_reclaim_mon_attrs.aggr_interval,
 168                 .max_age_region = UINT_MAX,
 169         };
 170
 171         return damon_new_scheme(
 172                         &pattern,
 173                         /* page out those, as soon as found */
 174                         DAMOS_PAGEOUT,
 175                         /* for each aggregation interval */
 176                         0,
 177                         /* under the quota. */
 178                         &damon_reclaim_quota,
 179                         /* (De)activate this according to the watermarks. */
 180                         &damon_reclaim_wmarks,
 181                         NUMA_NO_NODE);
 182 }
 183
 184 static int damon_reclaim_apply_parameters(void)
 185 {
 186         struct damon_ctx *param_ctx;
 187         struct damon_target *param_target;
 188         struct damos *scheme;
 189         struct damos_quota_goal *goal;
 190         struct damos_filter *filter;
 191         int err;
 192
 193         err = damon_modules_new_paddr_ctx_target(&param_ctx, &param_target);
 194         if (err)
 195                 return err;
 196
 197         err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
 198         if (err)
 199                 goto out;
 200
 201         err = -ENOMEM;
 202         scheme = damon_reclaim_new_scheme();
 203         if (!scheme)
 204                 goto out;
 205         damon_set_schemes(ctx, &scheme, 1);
 206
 207         if (quota_mem_pressure_us) {
 208                 goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US,
 209                                 quota_mem_pressure_us);
 210                 if (!goal)
 211                         goto out;
 212                 damos_add_quota_goal(&scheme->quota, goal);
 213         }
 214
 215         if (quota_autotune_feedback) {
 216                 goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000);
 217                 if (!goal)
 218                         goto out;
 219                 goal->current_value = quota_autotune_feedback;
 220                 damos_add_quota_goal(&scheme->quota, goal);
 221         }
 222
 223         if (skip_anon) {
 224                 filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true);
 225                 if (!filter)
 226                         goto out;
 227                 damos_add_filter(scheme, filter);
 228         }
 229
 230         err = damon_set_region_biggest_system_ram_default(param_target,
 231                                         &monitor_region_start,
 232                                         &monitor_region_end);
 233         if (err)
 234                 goto out;
 235         err = damon_commit_ctx(ctx, param_ctx);
 236 out:
 237         damon_destroy_ctx(param_ctx);
 238         return err;
 239 }
 240
 241 static int damon_reclaim_turn(bool on)
 242 {
 243         int err;
 244
 245         if (!on) {
 246                 err = damon_stop(&ctx, 1);
 247                 if (!err)
 248                         kdamond_pid = -1;
 249                 return err;
 250         }
 251
 252         err = damon_reclaim_apply_parameters();
 253         if (err)
 254                 return err;
 255
 256         err = damon_start(&ctx, 1, true);
 257         if (err)
 258                 return err;
 259         kdamond_pid = ctx->kdamond->pid;
 260         return 0;
 261 }
 262
 263 static int damon_reclaim_enabled_store(const char *val,
 264                 const struct kernel_param *kp)
 265 {
 266         bool is_enabled = enabled;
 267         bool enable;
 268         int err;
 269
 270         err = kstrtobool(val, &enable);
 271         if (err)
 272                 return err;
 273
 274         if (is_enabled == enable)
 275                 return 0;
 276
 277         /* Called before init function.  The function will handle this. */
 278         if (!ctx)
 279                 goto set_param_out;
 280
 281         err = damon_reclaim_turn(enable);
 282         if (err)
 283                 return err;
 284
 285 set_param_out:
 286         enabled = enable;
 287         return err;
 288 }
 289
 290 static const struct kernel_param_ops enabled_param_ops = {
 291         .set = damon_reclaim_enabled_store,
 292         .get = param_get_bool,
 293 };
 294
 295 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
 296 MODULE_PARM_DESC(enabled,
 297         "Enable or disable DAMON_RECLAIM (default: disabled)");
 298
 299 static int damon_reclaim_handle_commit_inputs(void)
 300 {
 301         int err;
 302
 303         if (!commit_inputs)
 304                 return 0;
 305
 306         err = damon_reclaim_apply_parameters();
 307         commit_inputs = false;
 308         return err;
 309 }
 310
 311 static int damon_reclaim_after_aggregation(struct damon_ctx *c)
 312 {
 313         struct damos *s;
 314
 315         /* update the stats parameter */
 316         damon_for_each_scheme(s, c)
 317                 damon_reclaim_stat = s->stat;
 318
 319         return damon_reclaim_handle_commit_inputs();
 320 }
 321
 322 static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
 323 {
 324         return damon_reclaim_handle_commit_inputs();
 325 }
 326
 327 static int __init damon_reclaim_init(void)
 328 {
 329         int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
 330
 331         if (err)
 332                 return err;
 333
 334         ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
 335         ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
 336
 337         /* 'enabled' has set before this function, probably via command line */
 338         if (enabled)
 339                 err = damon_reclaim_turn(true);
 340
 341         return err;
 342 }
 343
 344 module_init(damon_reclaim_init);