mm/hugetlb_cgroup.c

   1 /*
   2  *
   3  * Copyright IBM Corporation, 2012
   4  * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
   5  *
   6  * Cgroup v2
   7  * Copyright (C) 2019 Red Hat, Inc.
   8  * Author: Giuseppe Scrivano <gscrivan@redhat.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify it
  11  * under the terms of version 2.1 of the GNU Lesser General Public License
  12  * as published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it would be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  17  *
  18  */
  19
  20 #include <linux/cgroup.h>
  21 #include <linux/page_counter.h>
  22 #include <linux/slab.h>
  23 #include <linux/hugetlb.h>
  24 #include <linux/hugetlb_cgroup.h>
  25
  26 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
  27 #define MEMFILE_IDX(val)        (((val) >> 16) & 0xffff)
  28 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
  29
  30 /* Use t->m[0] to encode the offset */
  31 #define MEMFILE_OFFSET(t, m0)   (((offsetof(t, m0) << 16) | sizeof_field(t, m0)))
  32 #define MEMFILE_OFFSET0(val)    (((val) >> 16) & 0xffff)
  33 #define MEMFILE_FIELD_SIZE(val) ((val) & 0xffff)
  34
  35 #define DFL_TMPL_SIZE           ARRAY_SIZE(hugetlb_dfl_tmpl)
  36 #define LEGACY_TMPL_SIZE        ARRAY_SIZE(hugetlb_legacy_tmpl)
  37
  38 static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
  39 static struct cftype *dfl_files;
  40 static struct cftype *legacy_files;
  41
  42 static inline struct page_counter *
  43 __hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,
  44                                      bool rsvd)
  45 {
  46         if (rsvd)
  47                 return &h_cg->rsvd_hugepage[idx];
  48         return &h_cg->hugepage[idx];
  49 }
  50
  51 static inline struct page_counter *
  52 hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)
  53 {
  54         return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);
  55 }
  56
  57 static inline struct page_counter *
  58 hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)
  59 {
  60         return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);
  61 }
  62
  63 static inline
  64 struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
  65 {
  66         return s ? container_of(s, struct hugetlb_cgroup, css) : NULL;
  67 }
  68
  69 static inline
  70 struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task)
  71 {
  72         return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id));
  73 }
  74
  75 static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)
  76 {
  77         return (h_cg == root_h_cgroup);
  78 }
  79
  80 static inline struct hugetlb_cgroup *
  81 parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)
  82 {
  83         return hugetlb_cgroup_from_css(h_cg->css.parent);
  84 }
  85
  86 static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)
  87 {
  88         struct hstate *h;
  89
  90         for_each_hstate(h) {
  91                 if (page_counter_read(
  92                     hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h))))
  93                         return true;
  94         }
  95         return false;
  96 }
  97
  98 static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
  99                                 struct hugetlb_cgroup *parent_h_cgroup)
 100 {
 101         int idx;
 102
 103         for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) {
 104                 struct page_counter *fault_parent = NULL;
 105                 struct page_counter *rsvd_parent = NULL;
 106                 unsigned long limit;
 107                 int ret;
 108
 109                 if (parent_h_cgroup) {
 110                         fault_parent = hugetlb_cgroup_counter_from_cgroup(
 111                                 parent_h_cgroup, idx);
 112                         rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd(
 113                                 parent_h_cgroup, idx);
 114                 }
 115                 page_counter_init(hugetlb_cgroup_counter_from_cgroup(h_cgroup,
 116                                                                      idx),
 117                                   fault_parent, false);
 118                 page_counter_init(
 119                         hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
 120                         rsvd_parent, false);
 121
 122                 limit = round_down(PAGE_COUNTER_MAX,
 123                                    pages_per_huge_page(&hstates[idx]));
 124
 125                 ret = page_counter_set_max(
 126                         hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx),
 127                         limit);
 128                 VM_BUG_ON(ret);
 129                 ret = page_counter_set_max(
 130                         hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
 131                         limit);
 132                 VM_BUG_ON(ret);
 133         }
 134 }
 135
 136 static void hugetlb_cgroup_free(struct hugetlb_cgroup *h_cgroup)
 137 {
 138         int node;
 139
 140         for_each_node(node)
 141                 kfree(h_cgroup->nodeinfo[node]);
 142         kfree(h_cgroup);
 143 }
 144
 145 static struct cgroup_subsys_state *
 146 hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 147 {
 148         struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css);
 149         struct hugetlb_cgroup *h_cgroup;
 150         int node;
 151
 152         h_cgroup = kzalloc(struct_size(h_cgroup, nodeinfo, nr_node_ids),
 153                            GFP_KERNEL);
 154
 155         if (!h_cgroup)
 156                 return ERR_PTR(-ENOMEM);
 157
 158         if (!parent_h_cgroup)
 159                 root_h_cgroup = h_cgroup;
 160
 161         /*
 162          * TODO: this routine can waste much memory for nodes which will
 163          * never be onlined. It's better to use memory hotplug callback
 164          * function.
 165          */
 166         for_each_node(node) {
 167                 /* Set node_to_alloc to NUMA_NO_NODE for offline nodes. */
 168                 int node_to_alloc =
 169                         node_state(node, N_NORMAL_MEMORY) ? node : NUMA_NO_NODE;
 170                 h_cgroup->nodeinfo[node] =
 171                         kzalloc_node(sizeof(struct hugetlb_cgroup_per_node),
 172                                      GFP_KERNEL, node_to_alloc);
 173                 if (!h_cgroup->nodeinfo[node])
 174                         goto fail_alloc_nodeinfo;
 175         }
 176
 177         hugetlb_cgroup_init(h_cgroup, parent_h_cgroup);
 178         return &h_cgroup->css;
 179
 180 fail_alloc_nodeinfo:
 181         hugetlb_cgroup_free(h_cgroup);
 182         return ERR_PTR(-ENOMEM);
 183 }
 184
 185 static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css)
 186 {
 187         hugetlb_cgroup_free(hugetlb_cgroup_from_css(css));
 188 }
 189
 190 /*
 191  * Should be called with hugetlb_lock held.
 192  * Since we are holding hugetlb_lock, pages cannot get moved from
 193  * active list or uncharged from the cgroup, So no need to get
 194  * page reference and test for page active here. This function
 195  * cannot fail.
 196  */
 197 static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,
 198                                        struct page *page)
 199 {
 200         unsigned int nr_pages;
 201         struct page_counter *counter;
 202         struct hugetlb_cgroup *page_hcg;
 203         struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);
 204         struct folio *folio = page_folio(page);
 205
 206         page_hcg = hugetlb_cgroup_from_folio(folio);
 207         /*
 208          * We can have pages in active list without any cgroup
 209          * ie, hugepage with less than 3 pages. We can safely
 210          * ignore those pages.
 211          */
 212         if (!page_hcg || page_hcg != h_cg)
 213                 goto out;
 214
 215         nr_pages = compound_nr(page);
 216         if (!parent) {
 217                 parent = root_h_cgroup;
 218                 /* root has no limit */
 219                 page_counter_charge(&parent->hugepage[idx], nr_pages);
 220         }
 221         counter = &h_cg->hugepage[idx];
 222         /* Take the pages off the local counter */
 223         page_counter_cancel(counter, nr_pages);
 224
 225         set_hugetlb_cgroup(folio, parent);
 226 out:
 227         return;
 228 }
 229
 230 /*
 231  * Force the hugetlb cgroup to empty the hugetlb resources by moving them to
 232  * the parent cgroup.
 233  */
 234 static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
 235 {
 236         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
 237         struct hstate *h;
 238         struct page *page;
 239
 240         do {
 241                 for_each_hstate(h) {
 242                         spin_lock_irq(&hugetlb_lock);
 243                         list_for_each_entry(page, &h->hugepage_activelist, lru)
 244                                 hugetlb_cgroup_move_parent(hstate_index(h), h_cg, page);
 245
 246                         spin_unlock_irq(&hugetlb_lock);
 247                 }
 248                 cond_resched();
 249         } while (hugetlb_cgroup_have_usage(h_cg));
 250 }
 251
 252 static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx,
 253                                  enum hugetlb_memory_event event)
 254 {
 255         atomic_long_inc(&hugetlb->events_local[idx][event]);
 256         cgroup_file_notify(&hugetlb->events_local_file[idx]);
 257
 258         do {
 259                 atomic_long_inc(&hugetlb->events[idx][event]);
 260                 cgroup_file_notify(&hugetlb->events_file[idx]);
 261         } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) &&
 262                  !hugetlb_cgroup_is_root(hugetlb));
 263 }
 264
 265 static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
 266                                           struct hugetlb_cgroup **ptr,
 267                                           bool rsvd)
 268 {
 269         int ret = 0;
 270         struct page_counter *counter;
 271         struct hugetlb_cgroup *h_cg = NULL;
 272
 273         if (hugetlb_cgroup_disabled())
 274                 goto done;
 275 again:
 276         rcu_read_lock();
 277         h_cg = hugetlb_cgroup_from_task(current);
 278         if (!css_tryget(&h_cg->css)) {
 279                 rcu_read_unlock();
 280                 goto again;
 281         }
 282         rcu_read_unlock();
 283
 284         if (!page_counter_try_charge(
 285                     __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
 286                     nr_pages, &counter)) {
 287                 ret = -ENOMEM;
 288                 hugetlb_event(h_cg, idx, HUGETLB_MAX);
 289                 css_put(&h_cg->css);
 290                 goto done;
 291         }
 292         /* Reservations take a reference to the css because they do not get
 293          * reparented.
 294          */
 295         if (!rsvd)
 296                 css_put(&h_cg->css);
 297 done:
 298         *ptr = h_cg;
 299         return ret;
 300 }
 301
 302 int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
 303                                  struct hugetlb_cgroup **ptr)
 304 {
 305         return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false);
 306 }
 307
 308 int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,
 309                                       struct hugetlb_cgroup **ptr)
 310 {
 311         return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true);
 312 }
 313
 314 /* Should be called with hugetlb_lock held */
 315 static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
 316                                            struct hugetlb_cgroup *h_cg,
 317                                            struct folio *folio, bool rsvd)
 318 {
 319         if (hugetlb_cgroup_disabled() || !h_cg)
 320                 return;
 321         lockdep_assert_held(&hugetlb_lock);
 322         __set_hugetlb_cgroup(folio, h_cg, rsvd);
 323         if (!rsvd) {
 324                 unsigned long usage =
 325                         h_cg->nodeinfo[folio_nid(folio)]->usage[idx];
 326                 /*
 327                  * This write is not atomic due to fetching usage and writing
 328                  * to it, but that's fine because we call this with
 329                  * hugetlb_lock held anyway.
 330                  */
 331                 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],
 332                            usage + nr_pages);
 333         }
 334 }
 335
 336 void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
 337                                   struct hugetlb_cgroup *h_cg,
 338                                   struct folio *folio)
 339 {
 340         __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, false);
 341 }
 342
 343 void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
 344                                        struct hugetlb_cgroup *h_cg,
 345                                        struct folio *folio)
 346 {
 347         __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, folio, true);
 348 }
 349
 350 /*
 351  * Should be called with hugetlb_lock held
 352  */
 353 static void __hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,
 354                                            struct folio *folio, bool rsvd)
 355 {
 356         struct hugetlb_cgroup *h_cg;
 357
 358         if (hugetlb_cgroup_disabled())
 359                 return;
 360         lockdep_assert_held(&hugetlb_lock);
 361         h_cg = __hugetlb_cgroup_from_folio(folio, rsvd);
 362         if (unlikely(!h_cg))
 363                 return;
 364         __set_hugetlb_cgroup(folio, NULL, rsvd);
 365
 366         page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
 367                                                                    rsvd),
 368                               nr_pages);
 369
 370         if (rsvd)
 371                 css_put(&h_cg->css);
 372         else {
 373                 unsigned long usage =
 374                         h_cg->nodeinfo[folio_nid(folio)]->usage[idx];
 375                 /*
 376                  * This write is not atomic due to fetching usage and writing
 377                  * to it, but that's fine because we call this with
 378                  * hugetlb_lock held anyway.
 379                  */
 380                 WRITE_ONCE(h_cg->nodeinfo[folio_nid(folio)]->usage[idx],
 381                            usage - nr_pages);
 382         }
 383 }
 384
 385 void hugetlb_cgroup_uncharge_folio(int idx, unsigned long nr_pages,
 386                                   struct folio *folio)
 387 {
 388         __hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, false);
 389 }
 390
 391 void hugetlb_cgroup_uncharge_folio_rsvd(int idx, unsigned long nr_pages,
 392                                        struct folio *folio)
 393 {
 394         __hugetlb_cgroup_uncharge_folio(idx, nr_pages, folio, true);
 395 }
 396
 397 static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
 398                                              struct hugetlb_cgroup *h_cg,
 399                                              bool rsvd)
 400 {
 401         if (hugetlb_cgroup_disabled() || !h_cg)
 402                 return;
 403
 404         page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
 405                                                                    rsvd),
 406                               nr_pages);
 407
 408         if (rsvd)
 409                 css_put(&h_cg->css);
 410 }
 411
 412 void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
 413                                     struct hugetlb_cgroup *h_cg)
 414 {
 415         __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);
 416 }
 417
 418 void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,
 419                                          struct hugetlb_cgroup *h_cg)
 420 {
 421         __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);
 422 }
 423
 424 void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start,
 425                                      unsigned long end)
 426 {
 427         if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter ||
 428             !resv->css)
 429                 return;
 430
 431         page_counter_uncharge(resv->reservation_counter,
 432                               (end - start) * resv->pages_per_hpage);
 433         css_put(resv->css);
 434 }
 435
 436 void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
 437                                          struct file_region *rg,
 438                                          unsigned long nr_pages,
 439                                          bool region_del)
 440 {
 441         if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages)
 442                 return;
 443
 444         if (rg->reservation_counter && resv->pages_per_hpage &&
 445             !resv->reservation_counter) {
 446                 page_counter_uncharge(rg->reservation_counter,
 447                                       nr_pages * resv->pages_per_hpage);
 448                 /*
 449                  * Only do css_put(rg->css) when we delete the entire region
 450                  * because one file_region must hold exactly one css reference.
 451                  */
 452                 if (region_del)
 453                         css_put(rg->css);
 454         }
 455 }
 456
 457 enum {
 458         RES_USAGE,
 459         RES_RSVD_USAGE,
 460         RES_LIMIT,
 461         RES_RSVD_LIMIT,
 462         RES_MAX_USAGE,
 463         RES_RSVD_MAX_USAGE,
 464         RES_FAILCNT,
 465         RES_RSVD_FAILCNT,
 466 };
 467
 468 static int hugetlb_cgroup_read_numa_stat(struct seq_file *seq, void *dummy)
 469 {
 470         int nid;
 471         struct cftype *cft = seq_cft(seq);
 472         int idx = MEMFILE_IDX(cft->private);
 473         bool legacy = !cgroup_subsys_on_dfl(hugetlb_cgrp_subsys);
 474         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
 475         struct cgroup_subsys_state *css;
 476         unsigned long usage;
 477
 478         if (legacy) {
 479                 /* Add up usage across all nodes for the non-hierarchical total. */
 480                 usage = 0;
 481                 for_each_node_state(nid, N_MEMORY)
 482                         usage += READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]);
 483                 seq_printf(seq, "total=%lu", usage * PAGE_SIZE);
 484
 485                 /* Simply print the per-node usage for the non-hierarchical total. */
 486                 for_each_node_state(nid, N_MEMORY)
 487                         seq_printf(seq, " N%d=%lu", nid,
 488                                    READ_ONCE(h_cg->nodeinfo[nid]->usage[idx]) *
 489                                            PAGE_SIZE);
 490                 seq_putc(seq, '\n');
 491         }
 492
 493         /*
 494          * The hierarchical total is pretty much the value recorded by the
 495          * counter, so use that.
 496          */
 497         seq_printf(seq, "%stotal=%lu", legacy ? "hierarchical_" : "",
 498                    page_counter_read(&h_cg->hugepage[idx]) * PAGE_SIZE);
 499
 500         /*
 501          * For each node, transverse the css tree to obtain the hierarchical
 502          * node usage.
 503          */
 504         for_each_node_state(nid, N_MEMORY) {
 505                 usage = 0;
 506                 rcu_read_lock();
 507                 css_for_each_descendant_pre(css, &h_cg->css) {
 508                         usage += READ_ONCE(hugetlb_cgroup_from_css(css)
 509                                                    ->nodeinfo[nid]
 510                                                    ->usage[idx]);
 511                 }
 512                 rcu_read_unlock();
 513                 seq_printf(seq, " N%d=%lu", nid, usage * PAGE_SIZE);
 514         }
 515
 516         seq_putc(seq, '\n');
 517
 518         return 0;
 519 }
 520
 521 static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
 522                                    struct cftype *cft)
 523 {
 524         struct page_counter *counter;
 525         struct page_counter *rsvd_counter;
 526         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
 527
 528         counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
 529         rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)];
 530
 531         switch (MEMFILE_ATTR(cft->private)) {
 532         case RES_USAGE:
 533                 return (u64)page_counter_read(counter) * PAGE_SIZE;
 534         case RES_RSVD_USAGE:
 535                 return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE;
 536         case RES_LIMIT:
 537                 return (u64)counter->max * PAGE_SIZE;
 538         case RES_RSVD_LIMIT:
 539                 return (u64)rsvd_counter->max * PAGE_SIZE;
 540         case RES_MAX_USAGE:
 541                 return (u64)counter->watermark * PAGE_SIZE;
 542         case RES_RSVD_MAX_USAGE:
 543                 return (u64)rsvd_counter->watermark * PAGE_SIZE;
 544         case RES_FAILCNT:
 545                 return counter->failcnt;
 546         case RES_RSVD_FAILCNT:
 547                 return rsvd_counter->failcnt;
 548         default:
 549                 BUG();
 550         }
 551 }
 552
 553 static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v)
 554 {
 555         int idx;
 556         u64 val;
 557         struct cftype *cft = seq_cft(seq);
 558         unsigned long limit;
 559         struct page_counter *counter;
 560         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
 561
 562         idx = MEMFILE_IDX(cft->private);
 563         counter = &h_cg->hugepage[idx];
 564
 565         limit = round_down(PAGE_COUNTER_MAX,
 566                            pages_per_huge_page(&hstates[idx]));
 567
 568         switch (MEMFILE_ATTR(cft->private)) {
 569         case RES_RSVD_USAGE:
 570                 counter = &h_cg->rsvd_hugepage[idx];
 571                 fallthrough;
 572         case RES_USAGE:
 573                 val = (u64)page_counter_read(counter);
 574                 seq_printf(seq, "%llu\n", val * PAGE_SIZE);
 575                 break;
 576         case RES_RSVD_LIMIT:
 577                 counter = &h_cg->rsvd_hugepage[idx];
 578                 fallthrough;
 579         case RES_LIMIT:
 580                 val = (u64)counter->max;
 581                 if (val == limit)
 582                         seq_puts(seq, "max\n");
 583                 else
 584                         seq_printf(seq, "%llu\n", val * PAGE_SIZE);
 585                 break;
 586         default:
 587                 BUG();
 588         }
 589
 590         return 0;
 591 }
 592
 593 static DEFINE_MUTEX(hugetlb_limit_mutex);
 594
 595 static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
 596                                     char *buf, size_t nbytes, loff_t off,
 597                                     const char *max)
 598 {
 599         int ret, idx;
 600         unsigned long nr_pages;
 601         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
 602         bool rsvd = false;
 603
 604         if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
 605                 return -EINVAL;
 606
 607         buf = strstrip(buf);
 608         ret = page_counter_memparse(buf, max, &nr_pages);
 609         if (ret)
 610                 return ret;
 611
 612         idx = MEMFILE_IDX(of_cft(of)->private);
 613         nr_pages = round_down(nr_pages, pages_per_huge_page(&hstates[idx]));
 614
 615         switch (MEMFILE_ATTR(of_cft(of)->private)) {
 616         case RES_RSVD_LIMIT:
 617                 rsvd = true;
 618                 fallthrough;
 619         case RES_LIMIT:
 620                 mutex_lock(&hugetlb_limit_mutex);
 621                 ret = page_counter_set_max(
 622                         __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
 623                         nr_pages);
 624                 mutex_unlock(&hugetlb_limit_mutex);
 625                 break;
 626         default:
 627                 ret = -EINVAL;
 628                 break;
 629         }
 630         return ret ?: nbytes;
 631 }
 632
 633 static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of,
 634                                            char *buf, size_t nbytes, loff_t off)
 635 {
 636         return hugetlb_cgroup_write(of, buf, nbytes, off, "-1");
 637 }
 638
 639 static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of,
 640                                         char *buf, size_t nbytes, loff_t off)
 641 {
 642         return hugetlb_cgroup_write(of, buf, nbytes, off, "max");
 643 }
 644
 645 static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
 646                                     char *buf, size_t nbytes, loff_t off)
 647 {
 648         int ret = 0;
 649         struct page_counter *counter, *rsvd_counter;
 650         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
 651
 652         counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
 653         rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)];
 654
 655         switch (MEMFILE_ATTR(of_cft(of)->private)) {
 656         case RES_MAX_USAGE:
 657                 page_counter_reset_watermark(counter);
 658                 break;
 659         case RES_RSVD_MAX_USAGE:
 660                 page_counter_reset_watermark(rsvd_counter);
 661                 break;
 662         case RES_FAILCNT:
 663                 counter->failcnt = 0;
 664                 break;
 665         case RES_RSVD_FAILCNT:
 666                 rsvd_counter->failcnt = 0;
 667                 break;
 668         default:
 669                 ret = -EINVAL;
 670                 break;
 671         }
 672         return ret ?: nbytes;
 673 }
 674
 675 static char *mem_fmt(char *buf, int size, unsigned long hsize)
 676 {
 677         if (hsize >= SZ_1G)
 678                 snprintf(buf, size, "%luGB", hsize / SZ_1G);
 679         else if (hsize >= SZ_1M)
 680                 snprintf(buf, size, "%luMB", hsize / SZ_1M);
 681         else
 682                 snprintf(buf, size, "%luKB", hsize / SZ_1K);
 683         return buf;
 684 }
 685
 686 static int __hugetlb_events_show(struct seq_file *seq, bool local)
 687 {
 688         int idx;
 689         long max;
 690         struct cftype *cft = seq_cft(seq);
 691         struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
 692
 693         idx = MEMFILE_IDX(cft->private);
 694
 695         if (local)
 696                 max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);
 697         else
 698                 max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);
 699
 700         seq_printf(seq, "max %lu\n", max);
 701
 702         return 0;
 703 }
 704
 705 static int hugetlb_events_show(struct seq_file *seq, void *v)
 706 {
 707         return __hugetlb_events_show(seq, false);
 708 }
 709
 710 static int hugetlb_events_local_show(struct seq_file *seq, void *v)
 711 {
 712         return __hugetlb_events_show(seq, true);
 713 }
 714
 715 static struct cftype hugetlb_dfl_tmpl[] = {
 716         {
 717                 .name = "max",
 718                 .private = RES_LIMIT,
 719                 .seq_show = hugetlb_cgroup_read_u64_max,
 720                 .write = hugetlb_cgroup_write_dfl,
 721                 .flags = CFTYPE_NOT_ON_ROOT,
 722         },
 723         {
 724                 .name = "rsvd.max",
 725                 .private = RES_RSVD_LIMIT,
 726                 .seq_show = hugetlb_cgroup_read_u64_max,
 727                 .write = hugetlb_cgroup_write_dfl,
 728                 .flags = CFTYPE_NOT_ON_ROOT,
 729         },
 730         {
 731                 .name = "current",
 732                 .private = RES_USAGE,
 733                 .seq_show = hugetlb_cgroup_read_u64_max,
 734                 .flags = CFTYPE_NOT_ON_ROOT,
 735         },
 736         {
 737                 .name = "rsvd.current",
 738                 .private = RES_RSVD_USAGE,
 739                 .seq_show = hugetlb_cgroup_read_u64_max,
 740                 .flags = CFTYPE_NOT_ON_ROOT,
 741         },
 742         {
 743                 .name = "events",
 744                 .seq_show = hugetlb_events_show,
 745                 .file_offset = MEMFILE_OFFSET(struct hugetlb_cgroup, events_file[0]),
 746                 .flags = CFTYPE_NOT_ON_ROOT,
 747         },
 748         {
 749                 .name = "events.local",
 750                 .seq_show = hugetlb_events_local_show,
 751                 .file_offset = MEMFILE_OFFSET(struct hugetlb_cgroup, events_local_file[0]),
 752                 .flags = CFTYPE_NOT_ON_ROOT,
 753         },
 754         {
 755                 .name = "numa_stat",
 756                 .seq_show = hugetlb_cgroup_read_numa_stat,
 757                 .flags = CFTYPE_NOT_ON_ROOT,
 758         },
 759         /* don't need terminator here */
 760 };
 761
 762 static struct cftype hugetlb_legacy_tmpl[] = {
 763         {
 764                 .name = "limit_in_bytes",
 765                 .private = RES_LIMIT,
 766                 .read_u64 = hugetlb_cgroup_read_u64,
 767                 .write = hugetlb_cgroup_write_legacy,
 768         },
 769         {
 770                 .name = "rsvd.limit_in_bytes",
 771                 .private = RES_RSVD_LIMIT,
 772                 .read_u64 = hugetlb_cgroup_read_u64,
 773                 .write = hugetlb_cgroup_write_legacy,
 774         },
 775         {
 776                 .name = "usage_in_bytes",
 777                 .private = RES_USAGE,
 778                 .read_u64 = hugetlb_cgroup_read_u64,
 779         },
 780         {
 781                 .name = "rsvd.usage_in_bytes",
 782                 .private = RES_RSVD_USAGE,
 783                 .read_u64 = hugetlb_cgroup_read_u64,
 784         },
 785         {
 786                 .name = "max_usage_in_bytes",
 787                 .private = RES_MAX_USAGE,
 788                 .write = hugetlb_cgroup_reset,
 789                 .read_u64 = hugetlb_cgroup_read_u64,
 790         },
 791         {
 792                 .name = "rsvd.max_usage_in_bytes",
 793                 .private = RES_RSVD_MAX_USAGE,
 794                 .write = hugetlb_cgroup_reset,
 795                 .read_u64 = hugetlb_cgroup_read_u64,
 796         },
 797         {
 798                 .name = "failcnt",
 799                 .private = RES_FAILCNT,
 800                 .write = hugetlb_cgroup_reset,
 801                 .read_u64 = hugetlb_cgroup_read_u64,
 802         },
 803         {
 804                 .name = "rsvd.failcnt",
 805                 .private = RES_RSVD_FAILCNT,
 806                 .write = hugetlb_cgroup_reset,
 807                 .read_u64 = hugetlb_cgroup_read_u64,
 808         },
 809         {
 810                 .name = "numa_stat",
 811                 .seq_show = hugetlb_cgroup_read_numa_stat,
 812         },
 813         /* don't need terminator here */
 814 };
 815
 816 static void __init
 817 hugetlb_cgroup_cfttypes_init(struct hstate *h, struct cftype *cft,
 818                              struct cftype *tmpl, int tmpl_size)
 819 {
 820         char buf[32];
 821         int i, idx = hstate_index(h);
 822
 823         /* format the size */
 824         mem_fmt(buf, sizeof(buf), huge_page_size(h));
 825
 826         for (i = 0; i < tmpl_size; cft++, tmpl++, i++) {
 827                 *cft = *tmpl;
 828                 /* rebuild the name */
 829                 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.%s", buf, tmpl->name);
 830                 /* rebuild the private */
 831                 cft->private = MEMFILE_PRIVATE(idx, tmpl->private);
 832                 /* rebuild the file_offset */
 833                 if (tmpl->file_offset) {
 834                         unsigned int offset = tmpl->file_offset;
 835
 836                         cft->file_offset = MEMFILE_OFFSET0(offset) +
 837                                            MEMFILE_FIELD_SIZE(offset) * idx;
 838                 }
 839
 840                 lockdep_register_key(&cft->lockdep_key);
 841         }
 842 }
 843
 844 static void __init __hugetlb_cgroup_file_dfl_init(struct hstate *h)
 845 {
 846         int idx = hstate_index(h);
 847
 848         hugetlb_cgroup_cfttypes_init(h, dfl_files + idx * DFL_TMPL_SIZE,
 849                                      hugetlb_dfl_tmpl, DFL_TMPL_SIZE);
 850 }
 851
 852 static void __init __hugetlb_cgroup_file_legacy_init(struct hstate *h)
 853 {
 854         int idx = hstate_index(h);
 855
 856         hugetlb_cgroup_cfttypes_init(h, legacy_files + idx * LEGACY_TMPL_SIZE,
 857                                      hugetlb_legacy_tmpl, LEGACY_TMPL_SIZE);
 858 }
 859
 860 static void __init __hugetlb_cgroup_file_init(struct hstate *h)
 861 {
 862         __hugetlb_cgroup_file_dfl_init(h);
 863         __hugetlb_cgroup_file_legacy_init(h);
 864 }
 865
 866 static void __init __hugetlb_cgroup_file_pre_init(void)
 867 {
 868         int cft_count;
 869
 870         cft_count = hugetlb_max_hstate * DFL_TMPL_SIZE + 1; /* add terminator */
 871         dfl_files = kcalloc(cft_count, sizeof(struct cftype), GFP_KERNEL);
 872         BUG_ON(!dfl_files);
 873         cft_count = hugetlb_max_hstate * LEGACY_TMPL_SIZE + 1; /* add terminator */
 874         legacy_files = kcalloc(cft_count, sizeof(struct cftype), GFP_KERNEL);
 875         BUG_ON(!legacy_files);
 876 }
 877
 878 static void __init __hugetlb_cgroup_file_post_init(void)
 879 {
 880         WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys,
 881                                        dfl_files));
 882         WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
 883                                           legacy_files));
 884 }
 885
 886 void __init hugetlb_cgroup_file_init(void)
 887 {
 888         struct hstate *h;
 889
 890         __hugetlb_cgroup_file_pre_init();
 891         for_each_hstate(h)
 892                 __hugetlb_cgroup_file_init(h);
 893         __hugetlb_cgroup_file_post_init();
 894 }
 895
 896 /*
 897  * hugetlb_lock will make sure a parallel cgroup rmdir won't happen
 898  * when we migrate hugepages
 899  */
 900 void hugetlb_cgroup_migrate(struct folio *old_folio, struct folio *new_folio)
 901 {
 902         struct hugetlb_cgroup *h_cg;
 903         struct hugetlb_cgroup *h_cg_rsvd;
 904         struct hstate *h = folio_hstate(old_folio);
 905
 906         if (hugetlb_cgroup_disabled())
 907                 return;
 908
 909         spin_lock_irq(&hugetlb_lock);
 910         h_cg = hugetlb_cgroup_from_folio(old_folio);
 911         h_cg_rsvd = hugetlb_cgroup_from_folio_rsvd(old_folio);
 912         set_hugetlb_cgroup(old_folio, NULL);
 913         set_hugetlb_cgroup_rsvd(old_folio, NULL);
 914
 915         /* move the h_cg details to new cgroup */
 916         set_hugetlb_cgroup(new_folio, h_cg);
 917         set_hugetlb_cgroup_rsvd(new_folio, h_cg_rsvd);
 918         list_move(&new_folio->lru, &h->hugepage_activelist);
 919         spin_unlock_irq(&hugetlb_lock);
 920         return;
 921 }
 922
 923 static struct cftype hugetlb_files[] = {
 924         {} /* terminate */
 925 };
 926
 927 struct cgroup_subsys hugetlb_cgrp_subsys = {
 928         .css_alloc      = hugetlb_cgroup_css_alloc,
 929         .css_offline    = hugetlb_cgroup_css_offline,
 930         .css_free       = hugetlb_cgroup_css_free,
 931         .dfl_cftypes    = hugetlb_files,
 932         .legacy_cftypes = hugetlb_files,
 933 };