lib/alloc_tag.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/alloc_tag.h>
   3 #include <linux/execmem.h>
   4 #include <linux/fs.h>
   5 #include <linux/gfp.h>
   6 #include <linux/kallsyms.h>
   7 #include <linux/module.h>
   8 #include <linux/page_ext.h>
   9 #include <linux/proc_fs.h>
  10 #include <linux/seq_buf.h>
  11 #include <linux/seq_file.h>
  12 #include <linux/vmalloc.h>
  13
  14 #define ALLOCINFO_FILE_NAME             "allocinfo"
  15 #define MODULE_ALLOC_TAG_VMAP_SIZE      (100000UL * sizeof(struct alloc_tag))
  16 #define SECTION_START(NAME)             (CODETAG_SECTION_START_PREFIX NAME)
  17 #define SECTION_STOP(NAME)              (CODETAG_SECTION_STOP_PREFIX NAME)
  18
  19 #ifdef CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT
  20 static bool mem_profiling_support = true;
  21 #else
  22 static bool mem_profiling_support;
  23 #endif
  24
  25 static struct codetag_type *alloc_tag_cttype;
  26
  27 DEFINE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
  28 EXPORT_SYMBOL(_shared_alloc_tag);
  29
  30 DEFINE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
  31                         mem_alloc_profiling_key);
  32 DEFINE_STATIC_KEY_FALSE(mem_profiling_compressed);
  33
  34 struct alloc_tag_kernel_section kernel_tags = { NULL, 0 };
  35 unsigned long alloc_tag_ref_mask;
  36 int alloc_tag_ref_offs;
  37
  38 struct allocinfo_private {
  39         struct codetag_iterator iter;
  40         bool print_header;
  41 };
  42
  43 static void *allocinfo_start(struct seq_file *m, loff_t *pos)
  44 {
  45         struct allocinfo_private *priv;
  46         struct codetag *ct;
  47         loff_t node = *pos;
  48
  49         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
  50         m->private = priv;
  51         if (!priv)
  52                 return NULL;
  53
  54         priv->print_header = (node == 0);
  55         codetag_lock_module_list(alloc_tag_cttype, true);
  56         priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
  57         while ((ct = codetag_next_ct(&priv->iter)) != NULL && node)
  58                 node--;
  59
  60         return ct ? priv : NULL;
  61 }
  62
  63 static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
  64 {
  65         struct allocinfo_private *priv = (struct allocinfo_private *)arg;
  66         struct codetag *ct = codetag_next_ct(&priv->iter);
  67
  68         (*pos)++;
  69         if (!ct)
  70                 return NULL;
  71
  72         return priv;
  73 }
  74
  75 static void allocinfo_stop(struct seq_file *m, void *arg)
  76 {
  77         struct allocinfo_private *priv = (struct allocinfo_private *)m->private;
  78
  79         if (priv) {
  80                 codetag_lock_module_list(alloc_tag_cttype, false);
  81                 kfree(priv);
  82         }
  83 }
  84
  85 static void print_allocinfo_header(struct seq_buf *buf)
  86 {
  87         /* Output format version, so we can change it. */
  88         seq_buf_printf(buf, "allocinfo - version: 1.0\n");
  89         seq_buf_printf(buf, "#     <size>  <calls> <tag info>\n");
  90 }
  91
  92 static void alloc_tag_to_text(struct seq_buf *out, struct codetag *ct)
  93 {
  94         struct alloc_tag *tag = ct_to_alloc_tag(ct);
  95         struct alloc_tag_counters counter = alloc_tag_read(tag);
  96         s64 bytes = counter.bytes;
  97
  98         seq_buf_printf(out, "%12lli %8llu ", bytes, counter.calls);
  99         codetag_to_text(out, ct);
 100         seq_buf_putc(out, ' ');
 101         seq_buf_putc(out, '\n');
 102 }
 103
 104 static int allocinfo_show(struct seq_file *m, void *arg)
 105 {
 106         struct allocinfo_private *priv = (struct allocinfo_private *)arg;
 107         char *bufp;
 108         size_t n = seq_get_buf(m, &bufp);
 109         struct seq_buf buf;
 110
 111         seq_buf_init(&buf, bufp, n);
 112         if (priv->print_header) {
 113                 print_allocinfo_header(&buf);
 114                 priv->print_header = false;
 115         }
 116         alloc_tag_to_text(&buf, priv->iter.ct);
 117         seq_commit(m, seq_buf_used(&buf));
 118         return 0;
 119 }
 120
 121 static const struct seq_operations allocinfo_seq_op = {
 122         .start  = allocinfo_start,
 123         .next   = allocinfo_next,
 124         .stop   = allocinfo_stop,
 125         .show   = allocinfo_show,
 126 };
 127
 128 size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep)
 129 {
 130         struct codetag_iterator iter;
 131         struct codetag *ct;
 132         struct codetag_bytes n;
 133         unsigned int i, nr = 0;
 134
 135         if (can_sleep)
 136                 codetag_lock_module_list(alloc_tag_cttype, true);
 137         else if (!codetag_trylock_module_list(alloc_tag_cttype))
 138                 return 0;
 139
 140         iter = codetag_get_ct_iter(alloc_tag_cttype);
 141         while ((ct = codetag_next_ct(&iter))) {
 142                 struct alloc_tag_counters counter = alloc_tag_read(ct_to_alloc_tag(ct));
 143
 144                 n.ct    = ct;
 145                 n.bytes = counter.bytes;
 146
 147                 for (i = 0; i < nr; i++)
 148                         if (n.bytes > tags[i].bytes)
 149                                 break;
 150
 151                 if (i < count) {
 152                         nr -= nr == count;
 153                         memmove(&tags[i + 1],
 154                                 &tags[i],
 155                                 sizeof(tags[0]) * (nr - i));
 156                         nr++;
 157                         tags[i] = n;
 158                 }
 159         }
 160
 161         codetag_lock_module_list(alloc_tag_cttype, false);
 162
 163         return nr;
 164 }
 165
 166 void pgalloc_tag_split(struct folio *folio, int old_order, int new_order)
 167 {
 168         int i;
 169         struct alloc_tag *tag;
 170         unsigned int nr_pages = 1 << new_order;
 171
 172         if (!mem_alloc_profiling_enabled())
 173                 return;
 174
 175         tag = pgalloc_tag_get(&folio->page);
 176         if (!tag)
 177                 return;
 178
 179         for (i = nr_pages; i < (1 << old_order); i += nr_pages) {
 180                 union pgtag_ref_handle handle;
 181                 union codetag_ref ref;
 182
 183                 if (get_page_tag_ref(folio_page(folio, i), &ref, &handle)) {
 184                         /* Set new reference to point to the original tag */
 185                         alloc_tag_ref_set(&ref, tag);
 186                         update_page_tag_ref(handle, &ref);
 187                         put_page_tag_ref(handle);
 188                 }
 189         }
 190 }
 191
 192 void pgalloc_tag_copy(struct folio *new, struct folio *old)
 193 {
 194         union pgtag_ref_handle handle;
 195         union codetag_ref ref;
 196         struct alloc_tag *tag;
 197
 198         tag = pgalloc_tag_get(&old->page);
 199         if (!tag)
 200                 return;
 201
 202         if (!get_page_tag_ref(&new->page, &ref, &handle))
 203                 return;
 204
 205         /* Clear the old ref to the original allocation tag. */
 206         clear_page_tag_ref(&old->page);
 207         /* Decrement the counters of the tag on get_new_folio. */
 208         alloc_tag_sub(&ref, folio_size(new));
 209         __alloc_tag_ref_set(&ref, tag);
 210         update_page_tag_ref(handle, &ref);
 211         put_page_tag_ref(handle);
 212 }
 213
 214 static void shutdown_mem_profiling(bool remove_file)
 215 {
 216         if (mem_alloc_profiling_enabled())
 217                 static_branch_disable(&mem_alloc_profiling_key);
 218
 219         if (!mem_profiling_support)
 220                 return;
 221
 222         if (remove_file)
 223                 remove_proc_entry(ALLOCINFO_FILE_NAME, NULL);
 224         mem_profiling_support = false;
 225 }
 226
 227 static void __init procfs_init(void)
 228 {
 229         if (!mem_profiling_support)
 230                 return;
 231
 232         if (!proc_create_seq(ALLOCINFO_FILE_NAME, 0400, NULL, &allocinfo_seq_op)) {
 233                 pr_err("Failed to create %s file\n", ALLOCINFO_FILE_NAME);
 234                 shutdown_mem_profiling(false);
 235         }
 236 }
 237
 238 void __init alloc_tag_sec_init(void)
 239 {
 240         struct alloc_tag *last_codetag;
 241
 242         if (!mem_profiling_support)
 243                 return;
 244
 245         if (!static_key_enabled(&mem_profiling_compressed))
 246                 return;
 247
 248         kernel_tags.first_tag = (struct alloc_tag *)kallsyms_lookup_name(
 249                                         SECTION_START(ALLOC_TAG_SECTION_NAME));
 250         last_codetag = (struct alloc_tag *)kallsyms_lookup_name(
 251                                         SECTION_STOP(ALLOC_TAG_SECTION_NAME));
 252         kernel_tags.count = last_codetag - kernel_tags.first_tag;
 253
 254         /* Check if kernel tags fit into page flags */
 255         if (kernel_tags.count > (1UL << NR_UNUSED_PAGEFLAG_BITS)) {
 256                 shutdown_mem_profiling(false); /* allocinfo file does not exist yet */
 257                 pr_err("%lu allocation tags cannot be references using %d available page flag bits. Memory allocation profiling is disabled!\n",
 258                         kernel_tags.count, NR_UNUSED_PAGEFLAG_BITS);
 259                 return;
 260         }
 261
 262         alloc_tag_ref_offs = (LRU_REFS_PGOFF - NR_UNUSED_PAGEFLAG_BITS);
 263         alloc_tag_ref_mask = ((1UL << NR_UNUSED_PAGEFLAG_BITS) - 1);
 264         pr_debug("Memory allocation profiling compression is using %d page flag bits!\n",
 265                  NR_UNUSED_PAGEFLAG_BITS);
 266 }
 267
 268 #ifdef CONFIG_MODULES
 269
 270 static struct maple_tree mod_area_mt = MTREE_INIT(mod_area_mt, MT_FLAGS_ALLOC_RANGE);
 271 static struct vm_struct *vm_module_tags;
 272 /* A dummy object used to indicate an unloaded module */
 273 static struct module unloaded_mod;
 274 /* A dummy object used to indicate a module prepended area */
 275 static struct module prepend_mod;
 276
 277 struct alloc_tag_module_section module_tags;
 278
 279 static inline unsigned long alloc_tag_align(unsigned long val)
 280 {
 281         if (!static_key_enabled(&mem_profiling_compressed)) {
 282                 /* No alignment requirements when we are not indexing the tags */
 283                 return val;
 284         }
 285
 286         if (val % sizeof(struct alloc_tag) == 0)
 287                 return val;
 288         return ((val / sizeof(struct alloc_tag)) + 1) * sizeof(struct alloc_tag);
 289 }
 290
 291 static bool ensure_alignment(unsigned long align, unsigned int *prepend)
 292 {
 293         if (!static_key_enabled(&mem_profiling_compressed)) {
 294                 /* No alignment requirements when we are not indexing the tags */
 295                 return true;
 296         }
 297
 298         /*
 299          * If alloc_tag size is not a multiple of required alignment, tag
 300          * indexing does not work.
 301          */
 302         if (!IS_ALIGNED(sizeof(struct alloc_tag), align))
 303                 return false;
 304
 305         /* Ensure prepend consumes multiple of alloc_tag-sized blocks */
 306         if (*prepend)
 307                 *prepend = alloc_tag_align(*prepend);
 308
 309         return true;
 310 }
 311
 312 static inline bool tags_addressable(void)
 313 {
 314         unsigned long tag_idx_count;
 315
 316         if (!static_key_enabled(&mem_profiling_compressed))
 317                 return true; /* with page_ext tags are always addressable */
 318
 319         tag_idx_count = CODETAG_ID_FIRST + kernel_tags.count +
 320                         module_tags.size / sizeof(struct alloc_tag);
 321
 322         return tag_idx_count < (1UL << NR_UNUSED_PAGEFLAG_BITS);
 323 }
 324
 325 static bool needs_section_mem(struct module *mod, unsigned long size)
 326 {
 327         if (!mem_profiling_support)
 328                 return false;
 329
 330         return size >= sizeof(struct alloc_tag);
 331 }
 332
 333 static struct alloc_tag *find_used_tag(struct alloc_tag *from, struct alloc_tag *to)
 334 {
 335         while (from <= to) {
 336                 struct alloc_tag_counters counter;
 337
 338                 counter = alloc_tag_read(from);
 339                 if (counter.bytes)
 340                         return from;
 341                 from++;
 342         }
 343
 344         return NULL;
 345 }
 346
 347 /* Called with mod_area_mt locked */
 348 static void clean_unused_module_areas_locked(void)
 349 {
 350         MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
 351         struct module *val;
 352
 353         mas_for_each(&mas, val, module_tags.size) {
 354                 if (val != &unloaded_mod)
 355                         continue;
 356
 357                 /* Release area if all tags are unused */
 358                 if (!find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
 359                                    (struct alloc_tag *)(module_tags.start_addr + mas.last)))
 360                         mas_erase(&mas);
 361         }
 362 }
 363
 364 /* Called with mod_area_mt locked */
 365 static bool find_aligned_area(struct ma_state *mas, unsigned long section_size,
 366                               unsigned long size, unsigned int prepend, unsigned long align)
 367 {
 368         bool cleanup_done = false;
 369
 370 repeat:
 371         /* Try finding exact size and hope the start is aligned */
 372         if (!mas_empty_area(mas, 0, section_size - 1, prepend + size)) {
 373                 if (IS_ALIGNED(mas->index + prepend, align))
 374                         return true;
 375
 376                 /* Try finding larger area to align later */
 377                 mas_reset(mas);
 378                 if (!mas_empty_area(mas, 0, section_size - 1,
 379                                     size + prepend + align - 1))
 380                         return true;
 381         }
 382
 383         /* No free area, try cleanup stale data and repeat the search once */
 384         if (!cleanup_done) {
 385                 clean_unused_module_areas_locked();
 386                 cleanup_done = true;
 387                 mas_reset(mas);
 388                 goto repeat;
 389         }
 390
 391         return false;
 392 }
 393
 394 static int vm_module_tags_populate(void)
 395 {
 396         unsigned long phys_size = vm_module_tags->nr_pages << PAGE_SHIFT;
 397
 398         if (phys_size < module_tags.size) {
 399                 struct page **next_page = vm_module_tags->pages + vm_module_tags->nr_pages;
 400                 unsigned long addr = module_tags.start_addr + phys_size;
 401                 unsigned long more_pages;
 402                 unsigned long nr;
 403
 404                 more_pages = ALIGN(module_tags.size - phys_size, PAGE_SIZE) >> PAGE_SHIFT;
 405                 nr = alloc_pages_bulk_array_node(GFP_KERNEL | __GFP_NOWARN,
 406                                                  NUMA_NO_NODE, more_pages, next_page);
 407                 if (nr < more_pages ||
 408                     vmap_pages_range(addr, addr + (nr << PAGE_SHIFT), PAGE_KERNEL,
 409                                      next_page, PAGE_SHIFT) < 0) {
 410                         /* Clean up and error out */
 411                         for (int i = 0; i < nr; i++)
 412                                 __free_page(next_page[i]);
 413                         return -ENOMEM;
 414                 }
 415                 vm_module_tags->nr_pages += nr;
 416         }
 417
 418         return 0;
 419 }
 420
 421 static void *reserve_module_tags(struct module *mod, unsigned long size,
 422                                  unsigned int prepend, unsigned long align)
 423 {
 424         unsigned long section_size = module_tags.end_addr - module_tags.start_addr;
 425         MA_STATE(mas, &mod_area_mt, 0, section_size - 1);
 426         unsigned long offset;
 427         void *ret = NULL;
 428
 429         /* If no tags return error */
 430         if (size < sizeof(struct alloc_tag))
 431                 return ERR_PTR(-EINVAL);
 432
 433         /*
 434          * align is always power of 2, so we can use IS_ALIGNED and ALIGN.
 435          * align 0 or 1 means no alignment, to simplify set to 1.
 436          */
 437         if (!align)
 438                 align = 1;
 439
 440         if (!ensure_alignment(align, &prepend)) {
 441                 shutdown_mem_profiling(true);
 442                 pr_err("%s: alignment %lu is incompatible with allocation tag indexing. Memory allocation profiling is disabled!\n",
 443                         mod->name, align);
 444                 return ERR_PTR(-EINVAL);
 445         }
 446
 447         mas_lock(&mas);
 448         if (!find_aligned_area(&mas, section_size, size, prepend, align)) {
 449                 ret = ERR_PTR(-ENOMEM);
 450                 goto unlock;
 451         }
 452
 453         /* Mark found area as reserved */
 454         offset = mas.index;
 455         offset += prepend;
 456         offset = ALIGN(offset, align);
 457         if (offset != mas.index) {
 458                 unsigned long pad_start = mas.index;
 459
 460                 mas.last = offset - 1;
 461                 mas_store(&mas, &prepend_mod);
 462                 if (mas_is_err(&mas)) {
 463                         ret = ERR_PTR(xa_err(mas.node));
 464                         goto unlock;
 465                 }
 466                 mas.index = offset;
 467                 mas.last = offset + size - 1;
 468                 mas_store(&mas, mod);
 469                 if (mas_is_err(&mas)) {
 470                         mas.index = pad_start;
 471                         mas_erase(&mas);
 472                         ret = ERR_PTR(xa_err(mas.node));
 473                 }
 474         } else {
 475                 mas.last = offset + size - 1;
 476                 mas_store(&mas, mod);
 477                 if (mas_is_err(&mas))
 478                         ret = ERR_PTR(xa_err(mas.node));
 479         }
 480 unlock:
 481         mas_unlock(&mas);
 482
 483         if (IS_ERR(ret))
 484                 return ret;
 485
 486         if (module_tags.size < offset + size) {
 487                 int grow_res;
 488
 489                 module_tags.size = offset + size;
 490                 if (mem_alloc_profiling_enabled() && !tags_addressable()) {
 491                         shutdown_mem_profiling(true);
 492                         pr_warn("With module %s there are too many tags to fit in %d page flag bits. Memory allocation profiling is disabled!\n",
 493                                 mod->name, NR_UNUSED_PAGEFLAG_BITS);
 494                 }
 495
 496                 grow_res = vm_module_tags_populate();
 497                 if (grow_res) {
 498                         shutdown_mem_profiling(true);
 499                         pr_err("Failed to allocate memory for allocation tags in the module %s. Memory allocation profiling is disabled!\n",
 500                                mod->name);
 501                         return ERR_PTR(grow_res);
 502                 }
 503         }
 504
 505         return (struct alloc_tag *)(module_tags.start_addr + offset);
 506 }
 507
 508 static void release_module_tags(struct module *mod, bool used)
 509 {
 510         MA_STATE(mas, &mod_area_mt, module_tags.size, module_tags.size);
 511         struct alloc_tag *tag;
 512         struct module *val;
 513
 514         mas_lock(&mas);
 515         mas_for_each_rev(&mas, val, 0)
 516                 if (val == mod)
 517                         break;
 518
 519         if (!val) /* module not found */
 520                 goto out;
 521
 522         if (!used)
 523                 goto release_area;
 524
 525         /* Find out if the area is used */
 526         tag = find_used_tag((struct alloc_tag *)(module_tags.start_addr + mas.index),
 527                             (struct alloc_tag *)(module_tags.start_addr + mas.last));
 528         if (tag) {
 529                 struct alloc_tag_counters counter = alloc_tag_read(tag);
 530
 531                 pr_info("%s:%u module %s func:%s has %llu allocated at module unload\n",
 532                         tag->ct.filename, tag->ct.lineno, tag->ct.modname,
 533                         tag->ct.function, counter.bytes);
 534         } else {
 535                 used = false;
 536         }
 537 release_area:
 538         mas_store(&mas, used ? &unloaded_mod : NULL);
 539         val = mas_prev_range(&mas, 0);
 540         if (val == &prepend_mod)
 541                 mas_store(&mas, NULL);
 542 out:
 543         mas_unlock(&mas);
 544 }
 545
 546 static void replace_module(struct module *mod, struct module *new_mod)
 547 {
 548         MA_STATE(mas, &mod_area_mt, 0, module_tags.size);
 549         struct module *val;
 550
 551         mas_lock(&mas);
 552         mas_for_each(&mas, val, module_tags.size) {
 553                 if (val != mod)
 554                         continue;
 555
 556                 mas_store_gfp(&mas, new_mod, GFP_KERNEL);
 557                 break;
 558         }
 559         mas_unlock(&mas);
 560 }
 561
 562 static int __init alloc_mod_tags_mem(void)
 563 {
 564         /* Map space to copy allocation tags */
 565         vm_module_tags = execmem_vmap(MODULE_ALLOC_TAG_VMAP_SIZE);
 566         if (!vm_module_tags) {
 567                 pr_err("Failed to map %lu bytes for module allocation tags\n",
 568                         MODULE_ALLOC_TAG_VMAP_SIZE);
 569                 module_tags.start_addr = 0;
 570                 return -ENOMEM;
 571         }
 572
 573         vm_module_tags->pages = kmalloc_array(get_vm_area_size(vm_module_tags) >> PAGE_SHIFT,
 574                                         sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
 575         if (!vm_module_tags->pages) {
 576                 free_vm_area(vm_module_tags);
 577                 return -ENOMEM;
 578         }
 579
 580         module_tags.start_addr = (unsigned long)vm_module_tags->addr;
 581         module_tags.end_addr = module_tags.start_addr + MODULE_ALLOC_TAG_VMAP_SIZE;
 582         /* Ensure the base is alloc_tag aligned when required for indexing */
 583         module_tags.start_addr = alloc_tag_align(module_tags.start_addr);
 584
 585         return 0;
 586 }
 587
 588 static void __init free_mod_tags_mem(void)
 589 {
 590         int i;
 591
 592         module_tags.start_addr = 0;
 593         for (i = 0; i < vm_module_tags->nr_pages; i++)
 594                 __free_page(vm_module_tags->pages[i]);
 595         kfree(vm_module_tags->pages);
 596         free_vm_area(vm_module_tags);
 597 }
 598
 599 #else /* CONFIG_MODULES */
 600
 601 static inline int alloc_mod_tags_mem(void) { return 0; }
 602 static inline void free_mod_tags_mem(void) {}
 603
 604 #endif /* CONFIG_MODULES */
 605
 606 /* See: Documentation/mm/allocation-profiling.rst */
 607 static int __init setup_early_mem_profiling(char *str)
 608 {
 609         bool compressed = false;
 610         bool enable;
 611
 612         if (!str || !str[0])
 613                 return -EINVAL;
 614
 615         if (!strncmp(str, "never", 5)) {
 616                 enable = false;
 617                 mem_profiling_support = false;
 618                 pr_info("Memory allocation profiling is disabled!\n");
 619         } else {
 620                 char *token = strsep(&str, ",");
 621
 622                 if (kstrtobool(token, &enable))
 623                         return -EINVAL;
 624
 625                 if (str) {
 626
 627                         if (strcmp(str, "compressed"))
 628                                 return -EINVAL;
 629
 630                         compressed = true;
 631                 }
 632                 mem_profiling_support = true;
 633                 pr_info("Memory allocation profiling is enabled %s compression and is turned %s!\n",
 634                         compressed ? "with" : "without", enable ? "on" : "off");
 635         }
 636
 637         if (enable != mem_alloc_profiling_enabled()) {
 638                 if (enable)
 639                         static_branch_enable(&mem_alloc_profiling_key);
 640                 else
 641                         static_branch_disable(&mem_alloc_profiling_key);
 642         }
 643         if (compressed != static_key_enabled(&mem_profiling_compressed)) {
 644                 if (compressed)
 645                         static_branch_enable(&mem_profiling_compressed);
 646                 else
 647                         static_branch_disable(&mem_profiling_compressed);
 648         }
 649
 650         return 0;
 651 }
 652 early_param("sysctl.vm.mem_profiling", setup_early_mem_profiling);
 653
 654 static __init bool need_page_alloc_tagging(void)
 655 {
 656         if (static_key_enabled(&mem_profiling_compressed))
 657                 return false;
 658
 659         return mem_profiling_support;
 660 }
 661
 662 static __init void init_page_alloc_tagging(void)
 663 {
 664 }
 665
 666 struct page_ext_operations page_alloc_tagging_ops = {
 667         .size = sizeof(union codetag_ref),
 668         .need = need_page_alloc_tagging,
 669         .init = init_page_alloc_tagging,
 670 };
 671 EXPORT_SYMBOL(page_alloc_tagging_ops);
 672
 673 #ifdef CONFIG_SYSCTL
 674 static struct ctl_table memory_allocation_profiling_sysctls[] = {
 675         {
 676                 .procname       = "mem_profiling",
 677                 .data           = &mem_alloc_profiling_key,
 678 #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
 679                 .mode           = 0444,
 680 #else
 681                 .mode           = 0644,
 682 #endif
 683                 .proc_handler   = proc_do_static_key,
 684         },
 685 };
 686
 687 static void __init sysctl_init(void)
 688 {
 689         if (!mem_profiling_support)
 690                 memory_allocation_profiling_sysctls[0].mode = 0444;
 691
 692         register_sysctl_init("vm", memory_allocation_profiling_sysctls);
 693 }
 694 #else /* CONFIG_SYSCTL */
 695 static inline void sysctl_init(void) {}
 696 #endif /* CONFIG_SYSCTL */
 697
 698 static int __init alloc_tag_init(void)
 699 {
 700         const struct codetag_type_desc desc = {
 701                 .section                = ALLOC_TAG_SECTION_NAME,
 702                 .tag_size               = sizeof(struct alloc_tag),
 703 #ifdef CONFIG_MODULES
 704                 .needs_section_mem      = needs_section_mem,
 705                 .alloc_section_mem      = reserve_module_tags,
 706                 .free_section_mem       = release_module_tags,
 707                 .module_replaced        = replace_module,
 708 #endif
 709         };
 710         int res;
 711
 712         res = alloc_mod_tags_mem();
 713         if (res)
 714                 return res;
 715
 716         alloc_tag_cttype = codetag_register_type(&desc);
 717         if (IS_ERR(alloc_tag_cttype)) {
 718                 free_mod_tags_mem();
 719                 return PTR_ERR(alloc_tag_cttype);
 720         }
 721
 722         sysctl_init();
 723         procfs_init();
 724
 725         return 0;
 726 }
 727 module_init(alloc_tag_init);