mm/damon/paddr.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * DAMON Primitives for The Physical Address Space
   4  *
   5  * Author: SeongJae Park <sj@kernel.org>
   6  */
   7
   8 #define pr_fmt(fmt) "damon-pa: " fmt
   9
  10 #include <linux/mmu_notifier.h>
  11 #include <linux/page_idle.h>
  12 #include <linux/pagemap.h>
  13 #include <linux/rmap.h>
  14 #include <linux/swap.h>
  15 #include <linux/memory-tiers.h>
  16 #include <linux/migrate.h>
  17 #include <linux/mm_inline.h>
  18
  19 #include "../internal.h"
  20 #include "ops-common.h"
  21
  22 static bool damon_folio_mkold_one(struct folio *folio,
  23                 struct vm_area_struct *vma, unsigned long addr, void *arg)
  24 {
  25         DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
  26
  27         while (page_vma_mapped_walk(&pvmw)) {
  28                 addr = pvmw.address;
  29                 if (pvmw.pte)
  30                         damon_ptep_mkold(pvmw.pte, vma, addr);
  31                 else
  32                         damon_pmdp_mkold(pvmw.pmd, vma, addr);
  33         }
  34         return true;
  35 }
  36
  37 static void damon_folio_mkold(struct folio *folio)
  38 {
  39         struct rmap_walk_control rwc = {
  40                 .rmap_one = damon_folio_mkold_one,
  41                 .anon_lock = folio_lock_anon_vma_read,
  42         };
  43         bool need_lock;
  44
  45         if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
  46                 folio_set_idle(folio);
  47                 return;
  48         }
  49
  50         need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
  51         if (need_lock && !folio_trylock(folio))
  52                 return;
  53
  54         rmap_walk(folio, &rwc);
  55
  56         if (need_lock)
  57                 folio_unlock(folio);
  58
  59 }
  60
  61 static void damon_pa_mkold(unsigned long paddr)
  62 {
  63         struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
  64
  65         if (!folio)
  66                 return;
  67
  68         damon_folio_mkold(folio);
  69         folio_put(folio);
  70 }
  71
  72 static void __damon_pa_prepare_access_check(struct damon_region *r)
  73 {
  74         r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
  75
  76         damon_pa_mkold(r->sampling_addr);
  77 }
  78
  79 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
  80 {
  81         struct damon_target *t;
  82         struct damon_region *r;
  83
  84         damon_for_each_target(t, ctx) {
  85                 damon_for_each_region(r, t)
  86                         __damon_pa_prepare_access_check(r);
  87         }
  88 }
  89
  90 static bool damon_folio_young_one(struct folio *folio,
  91                 struct vm_area_struct *vma, unsigned long addr, void *arg)
  92 {
  93         bool *accessed = arg;
  94         DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
  95
  96         *accessed = false;
  97         while (page_vma_mapped_walk(&pvmw)) {
  98                 addr = pvmw.address;
  99                 if (pvmw.pte) {
 100                         *accessed = pte_young(ptep_get(pvmw.pte)) ||
 101                                 !folio_test_idle(folio) ||
 102                                 mmu_notifier_test_young(vma->vm_mm, addr);
 103                 } else {
 104 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 105                         *accessed = pmd_young(pmdp_get(pvmw.pmd)) ||
 106                                 !folio_test_idle(folio) ||
 107                                 mmu_notifier_test_young(vma->vm_mm, addr);
 108 #else
 109                         WARN_ON_ONCE(1);
 110 #endif  /* CONFIG_TRANSPARENT_HUGEPAGE */
 111                 }
 112                 if (*accessed) {
 113                         page_vma_mapped_walk_done(&pvmw);
 114                         break;
 115                 }
 116         }
 117
 118         /* If accessed, stop walking */
 119         return *accessed == false;
 120 }
 121
 122 static bool damon_folio_young(struct folio *folio)
 123 {
 124         bool accessed = false;
 125         struct rmap_walk_control rwc = {
 126                 .arg = &accessed,
 127                 .rmap_one = damon_folio_young_one,
 128                 .anon_lock = folio_lock_anon_vma_read,
 129         };
 130         bool need_lock;
 131
 132         if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
 133                 if (folio_test_idle(folio))
 134                         return false;
 135                 else
 136                         return true;
 137         }
 138
 139         need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
 140         if (need_lock && !folio_trylock(folio))
 141                 return false;
 142
 143         rmap_walk(folio, &rwc);
 144
 145         if (need_lock)
 146                 folio_unlock(folio);
 147
 148         return accessed;
 149 }
 150
 151 static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz)
 152 {
 153         struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
 154         bool accessed;
 155
 156         if (!folio)
 157                 return false;
 158
 159         accessed = damon_folio_young(folio);
 160         *folio_sz = folio_size(folio);
 161         folio_put(folio);
 162         return accessed;
 163 }
 164
 165 static void __damon_pa_check_access(struct damon_region *r,
 166                 struct damon_attrs *attrs)
 167 {
 168         static unsigned long last_addr;
 169         static unsigned long last_folio_sz = PAGE_SIZE;
 170         static bool last_accessed;
 171
 172         /* If the region is in the last checked page, reuse the result */
 173         if (ALIGN_DOWN(last_addr, last_folio_sz) ==
 174                                 ALIGN_DOWN(r->sampling_addr, last_folio_sz)) {
 175                 damon_update_region_access_rate(r, last_accessed, attrs);
 176                 return;
 177         }
 178
 179         last_accessed = damon_pa_young(r->sampling_addr, &last_folio_sz);
 180         damon_update_region_access_rate(r, last_accessed, attrs);
 181
 182         last_addr = r->sampling_addr;
 183 }
 184
 185 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
 186 {
 187         struct damon_target *t;
 188         struct damon_region *r;
 189         unsigned int max_nr_accesses = 0;
 190
 191         damon_for_each_target(t, ctx) {
 192                 damon_for_each_region(r, t) {
 193                         __damon_pa_check_access(r, &ctx->attrs);
 194                         max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
 195                 }
 196         }
 197
 198         return max_nr_accesses;
 199 }
 200
 201 static bool __damos_pa_filter_out(struct damos_filter *filter,
 202                 struct folio *folio)
 203 {
 204         bool matched = false;
 205         struct mem_cgroup *memcg;
 206
 207         switch (filter->type) {
 208         case DAMOS_FILTER_TYPE_ANON:
 209                 matched = folio_test_anon(folio);
 210                 break;
 211         case DAMOS_FILTER_TYPE_MEMCG:
 212                 rcu_read_lock();
 213                 memcg = folio_memcg_check(folio);
 214                 if (!memcg)
 215                         matched = false;
 216                 else
 217                         matched = filter->memcg_id == mem_cgroup_id(memcg);
 218                 rcu_read_unlock();
 219                 break;
 220         case DAMOS_FILTER_TYPE_YOUNG:
 221                 matched = damon_folio_young(folio);
 222                 if (matched)
 223                         damon_folio_mkold(folio);
 224                 break;
 225         default:
 226                 break;
 227         }
 228
 229         return matched == filter->matching;
 230 }
 231
 232 /*
 233  * damos_pa_filter_out - Return true if the page should be filtered out.
 234  */
 235 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio)
 236 {
 237         struct damos_filter *filter;
 238
 239         damos_for_each_filter(filter, scheme) {
 240                 if (__damos_pa_filter_out(filter, folio))
 241                         return true;
 242         }
 243         return false;
 244 }
 245
 246 static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s)
 247 {
 248         unsigned long addr, applied;
 249         LIST_HEAD(folio_list);
 250         bool install_young_filter = true;
 251         struct damos_filter *filter;
 252
 253         /* check access in page level again by default */
 254         damos_for_each_filter(filter, s) {
 255                 if (filter->type == DAMOS_FILTER_TYPE_YOUNG) {
 256                         install_young_filter = false;
 257                         break;
 258                 }
 259         }
 260         if (install_young_filter) {
 261                 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true);
 262                 if (!filter)
 263                         return 0;
 264                 damos_add_filter(s, filter);
 265         }
 266
 267         for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
 268                 struct folio *folio = damon_get_folio(PHYS_PFN(addr));
 269
 270                 if (!folio)
 271                         continue;
 272
 273                 if (damos_pa_filter_out(s, folio))
 274                         goto put_folio;
 275
 276                 folio_clear_referenced(folio);
 277                 folio_test_clear_young(folio);
 278                 if (!folio_isolate_lru(folio))
 279                         goto put_folio;
 280                 if (folio_test_unevictable(folio))
 281                         folio_putback_lru(folio);
 282                 else
 283                         list_add(&folio->lru, &folio_list);
 284 put_folio:
 285                 folio_put(folio);
 286         }
 287         if (install_young_filter)
 288                 damos_destroy_filter(filter);
 289         applied = reclaim_pages(&folio_list);
 290         cond_resched();
 291         return applied * PAGE_SIZE;
 292 }
 293
 294 static inline unsigned long damon_pa_mark_accessed_or_deactivate(
 295                 struct damon_region *r, struct damos *s, bool mark_accessed)
 296 {
 297         unsigned long addr, applied = 0;
 298
 299         for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
 300                 struct folio *folio = damon_get_folio(PHYS_PFN(addr));
 301
 302                 if (!folio)
 303                         continue;
 304
 305                 if (damos_pa_filter_out(s, folio))
 306                         goto put_folio;
 307
 308                 if (mark_accessed)
 309                         folio_mark_accessed(folio);
 310                 else
 311                         folio_deactivate(folio);
 312                 applied += folio_nr_pages(folio);
 313 put_folio:
 314                 folio_put(folio);
 315         }
 316         return applied * PAGE_SIZE;
 317 }
 318
 319 static unsigned long damon_pa_mark_accessed(struct damon_region *r,
 320         struct damos *s)
 321 {
 322         return damon_pa_mark_accessed_or_deactivate(r, s, true);
 323 }
 324
 325 static unsigned long damon_pa_deactivate_pages(struct damon_region *r,
 326         struct damos *s)
 327 {
 328         return damon_pa_mark_accessed_or_deactivate(r, s, false);
 329 }
 330
 331 static unsigned int __damon_pa_migrate_folio_list(
 332                 struct list_head *migrate_folios, struct pglist_data *pgdat,
 333                 int target_nid)
 334 {
 335         unsigned int nr_succeeded = 0;
 336         nodemask_t allowed_mask = NODE_MASK_NONE;
 337         struct migration_target_control mtc = {
 338                 /*
 339                  * Allocate from 'node', or fail quickly and quietly.
 340                  * When this happens, 'page' will likely just be discarded
 341                  * instead of migrated.
 342                  */
 343                 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
 344                         __GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT,
 345                 .nid = target_nid,
 346                 .nmask = &allowed_mask
 347         };
 348
 349         if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE)
 350                 return 0;
 351
 352         if (list_empty(migrate_folios))
 353                 return 0;
 354
 355         /* Migration ignores all cpuset and mempolicy settings */
 356         migrate_pages(migrate_folios, alloc_migrate_folio, NULL,
 357                       (unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON,
 358                       &nr_succeeded);
 359
 360         return nr_succeeded;
 361 }
 362
 363 static unsigned int damon_pa_migrate_folio_list(struct list_head *folio_list,
 364                                                 struct pglist_data *pgdat,
 365                                                 int target_nid)
 366 {
 367         unsigned int nr_migrated = 0;
 368         struct folio *folio;
 369         LIST_HEAD(ret_folios);
 370         LIST_HEAD(migrate_folios);
 371
 372         while (!list_empty(folio_list)) {
 373                 struct folio *folio;
 374
 375                 cond_resched();
 376
 377                 folio = lru_to_folio(folio_list);
 378                 list_del(&folio->lru);
 379
 380                 if (!folio_trylock(folio))
 381                         goto keep;
 382
 383                 /* Relocate its contents to another node. */
 384                 list_add(&folio->lru, &migrate_folios);
 385                 folio_unlock(folio);
 386                 continue;
 387 keep:
 388                 list_add(&folio->lru, &ret_folios);
 389         }
 390         /* 'folio_list' is always empty here */
 391
 392         /* Migrate folios selected for migration */
 393         nr_migrated += __damon_pa_migrate_folio_list(
 394                         &migrate_folios, pgdat, target_nid);
 395         /*
 396          * Folios that could not be migrated are still in @migrate_folios.  Add
 397          * those back on @folio_list
 398          */
 399         if (!list_empty(&migrate_folios))
 400                 list_splice_init(&migrate_folios, folio_list);
 401
 402         try_to_unmap_flush();
 403
 404         list_splice(&ret_folios, folio_list);
 405
 406         while (!list_empty(folio_list)) {
 407                 folio = lru_to_folio(folio_list);
 408                 list_del(&folio->lru);
 409                 folio_putback_lru(folio);
 410         }
 411
 412         return nr_migrated;
 413 }
 414
 415 static unsigned long damon_pa_migrate_pages(struct list_head *folio_list,
 416                                             int target_nid)
 417 {
 418         int nid;
 419         unsigned long nr_migrated = 0;
 420         LIST_HEAD(node_folio_list);
 421         unsigned int noreclaim_flag;
 422
 423         if (list_empty(folio_list))
 424                 return nr_migrated;
 425
 426         noreclaim_flag = memalloc_noreclaim_save();
 427
 428         nid = folio_nid(lru_to_folio(folio_list));
 429         do {
 430                 struct folio *folio = lru_to_folio(folio_list);
 431
 432                 if (nid == folio_nid(folio)) {
 433                         list_move(&folio->lru, &node_folio_list);
 434                         continue;
 435                 }
 436
 437                 nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,
 438                                                            NODE_DATA(nid),
 439                                                            target_nid);
 440                 nid = folio_nid(lru_to_folio(folio_list));
 441         } while (!list_empty(folio_list));
 442
 443         nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,
 444                                                    NODE_DATA(nid),
 445                                                    target_nid);
 446
 447         memalloc_noreclaim_restore(noreclaim_flag);
 448
 449         return nr_migrated;
 450 }
 451
 452 static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s)
 453 {
 454         unsigned long addr, applied;
 455         LIST_HEAD(folio_list);
 456
 457         for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
 458                 struct folio *folio = damon_get_folio(PHYS_PFN(addr));
 459
 460                 if (!folio)
 461                         continue;
 462
 463                 if (damos_pa_filter_out(s, folio))
 464                         goto put_folio;
 465
 466                 if (!folio_isolate_lru(folio))
 467                         goto put_folio;
 468                 list_add(&folio->lru, &folio_list);
 469 put_folio:
 470                 folio_put(folio);
 471         }
 472         applied = damon_pa_migrate_pages(&folio_list, s->target_nid);
 473         cond_resched();
 474         return applied * PAGE_SIZE;
 475 }
 476
 477
 478 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
 479                 struct damon_target *t, struct damon_region *r,
 480                 struct damos *scheme)
 481 {
 482         switch (scheme->action) {
 483         case DAMOS_PAGEOUT:
 484                 return damon_pa_pageout(r, scheme);
 485         case DAMOS_LRU_PRIO:
 486                 return damon_pa_mark_accessed(r, scheme);
 487         case DAMOS_LRU_DEPRIO:
 488                 return damon_pa_deactivate_pages(r, scheme);
 489         case DAMOS_MIGRATE_HOT:
 490         case DAMOS_MIGRATE_COLD:
 491                 return damon_pa_migrate(r, scheme);
 492         case DAMOS_STAT:
 493                 break;
 494         default:
 495                 /* DAMOS actions that not yet supported by 'paddr'. */
 496                 break;
 497         }
 498         return 0;
 499 }
 500
 501 static int damon_pa_scheme_score(struct damon_ctx *context,
 502                 struct damon_target *t, struct damon_region *r,
 503                 struct damos *scheme)
 504 {
 505         switch (scheme->action) {
 506         case DAMOS_PAGEOUT:
 507                 return damon_cold_score(context, r, scheme);
 508         case DAMOS_LRU_PRIO:
 509                 return damon_hot_score(context, r, scheme);
 510         case DAMOS_LRU_DEPRIO:
 511                 return damon_cold_score(context, r, scheme);
 512         case DAMOS_MIGRATE_HOT:
 513                 return damon_hot_score(context, r, scheme);
 514         case DAMOS_MIGRATE_COLD:
 515                 return damon_cold_score(context, r, scheme);
 516         default:
 517                 break;
 518         }
 519
 520         return DAMOS_MAX_SCORE;
 521 }
 522
 523 static int __init damon_pa_initcall(void)
 524 {
 525         struct damon_operations ops = {
 526                 .id = DAMON_OPS_PADDR,
 527                 .init = NULL,
 528                 .update = NULL,
 529                 .prepare_access_checks = damon_pa_prepare_access_checks,
 530                 .check_accesses = damon_pa_check_accesses,
 531                 .reset_aggregated = NULL,
 532                 .target_valid = NULL,
 533                 .cleanup = NULL,
 534                 .apply_scheme = damon_pa_apply_scheme,
 535                 .get_scheme_score = damon_pa_scheme_score,
 536         };
 537
 538         return damon_register_ops(&ops);
 539 };
 540
 541 subsys_initcall(damon_pa_initcall);