kernel/module/internal.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /* Module internals
   3  *
   4  * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
   7  */
   8
   9 #include <linux/elf.h>
  10 #include <linux/compiler.h>
  11 #include <linux/module.h>
  12 #include <linux/mutex.h>
  13 #include <linux/rculist.h>
  14 #include <linux/rcupdate.h>
  15 #include <linux/mm.h>
  16
  17 #ifndef ARCH_SHF_SMALL
  18 #define ARCH_SHF_SMALL 0
  19 #endif
  20
  21 /*
  22  * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
  23  * section. This leaves 28 bits for offset on 32-bit systems, which is
  24  * about 256 MiB (WARN_ON_ONCE if we exceed that).
  25  */
  26
  27 #define SH_ENTSIZE_TYPE_BITS    4
  28 #define SH_ENTSIZE_TYPE_SHIFT   (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
  29 #define SH_ENTSIZE_TYPE_MASK    ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
  30 #define SH_ENTSIZE_OFFSET_MASK  ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
  31
  32 /* Maximum number of characters written by module_flags() */
  33 #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
  34
  35 struct kernel_symbol {
  36 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
  37         int value_offset;
  38         int name_offset;
  39         int namespace_offset;
  40 #else
  41         unsigned long value;
  42         const char *name;
  43         const char *namespace;
  44 #endif
  45 };
  46
  47 extern struct mutex module_mutex;
  48 extern struct list_head modules;
  49
  50 extern struct module_attribute *modinfo_attrs[];
  51 extern size_t modinfo_attrs_count;
  52
  53 /* Provided by the linker */
  54 extern const struct kernel_symbol __start___ksymtab[];
  55 extern const struct kernel_symbol __stop___ksymtab[];
  56 extern const struct kernel_symbol __start___ksymtab_gpl[];
  57 extern const struct kernel_symbol __stop___ksymtab_gpl[];
  58 extern const s32 __start___kcrctab[];
  59 extern const s32 __start___kcrctab_gpl[];
  60
  61 struct load_info {
  62         const char *name;
  63         /* pointer to module in temporary copy, freed at end of load_module() */
  64         struct module *mod;
  65         Elf_Ehdr *hdr;
  66         unsigned long len;
  67         Elf_Shdr *sechdrs;
  68         char *secstrings, *strtab;
  69         unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
  70         bool sig_ok;
  71 #ifdef CONFIG_KALLSYMS
  72         unsigned long mod_kallsyms_init_off;
  73 #endif
  74 #ifdef CONFIG_MODULE_DECOMPRESS
  75 #ifdef CONFIG_MODULE_STATS
  76         unsigned long compressed_len;
  77 #endif
  78         struct page **pages;
  79         unsigned int max_pages;
  80         unsigned int used_pages;
  81 #endif
  82         struct {
  83                 unsigned int sym;
  84                 unsigned int str;
  85                 unsigned int mod;
  86                 unsigned int vers;
  87                 unsigned int info;
  88                 unsigned int pcpu;
  89         } index;
  90 };
  91
  92 enum mod_license {
  93         NOT_GPL_ONLY,
  94         GPL_ONLY,
  95 };
  96
  97 struct find_symbol_arg {
  98         /* Input */
  99         const char *name;
 100         bool gplok;
 101         bool warn;
 102
 103         /* Output */
 104         struct module *owner;
 105         const s32 *crc;
 106         const struct kernel_symbol *sym;
 107         enum mod_license license;
 108 };
 109
 110 int mod_verify_sig(const void *mod, struct load_info *info);
 111 int try_to_force_load(struct module *mod, const char *reason);
 112 bool find_symbol(struct find_symbol_arg *fsa);
 113 struct module *find_module_all(const char *name, size_t len, bool even_unformed);
 114 int cmp_name(const void *name, const void *sym);
 115 long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
 116                                 Elf_Shdr *sechdr, unsigned int section);
 117 char *module_flags(struct module *mod, char *buf, bool show_state);
 118 size_t module_flags_taint(unsigned long taints, char *buf);
 119
 120 char *module_next_tag_pair(char *string, unsigned long *secsize);
 121
 122 #define for_each_modinfo_entry(entry, info, name) \
 123         for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
 124
 125 static inline void module_assert_mutex_or_preempt(void)
 126 {
 127 #ifdef CONFIG_LOCKDEP
 128         if (unlikely(!debug_locks))
 129                 return;
 130
 131         WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
 132                      !lockdep_is_held(&module_mutex));
 133 #endif
 134 }
 135
 136 static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
 137 {
 138 #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
 139         return (unsigned long)offset_to_ptr(&sym->value_offset);
 140 #else
 141         return sym->value;
 142 #endif
 143 }
 144
 145 #ifdef CONFIG_LIVEPATCH
 146 int copy_module_elf(struct module *mod, struct load_info *info);
 147 void free_module_elf(struct module *mod);
 148 #else /* !CONFIG_LIVEPATCH */
 149 static inline int copy_module_elf(struct module *mod, struct load_info *info)
 150 {
 151         return 0;
 152 }
 153
 154 static inline void free_module_elf(struct module *mod) { }
 155 #endif /* CONFIG_LIVEPATCH */
 156
 157 static inline bool set_livepatch_module(struct module *mod)
 158 {
 159 #ifdef CONFIG_LIVEPATCH
 160         mod->klp = true;
 161         return true;
 162 #else
 163         return false;
 164 #endif
 165 }
 166
 167 /**
 168  * enum fail_dup_mod_reason - state at which a duplicate module was detected
 169  *
 170  * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
 171  *      we've determined that another module with the same name is already loaded
 172  *      or being processed on our &modules list. This happens on early_mod_check()
 173  *      right before layout_and_allocate(). The kernel would have already
 174  *      vmalloc()'d space for the entire module through finit_module(). If
 175  *      decompression was used two vmap() spaces were used. These failures can
 176  *      happen when userspace has not seen the module present on the kernel and
 177  *      tries to load the module multiple times at same time.
 178  * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
 179  *      checks and the kernel determines that the module was unique and because
 180  *      of this allocated yet another private kernel copy of the module space in
 181  *      layout_and_allocate() but after this determined in add_unformed_module()
 182  *      that another module with the same name is already loaded or being processed.
 183  *      These failures should be mitigated as much as possible and are indicative
 184  *      of really fast races in loading modules. Without module decompression
 185  *      they waste twice as much vmap space. With module decompression three
 186  *      times the module's size vmap space is wasted.
 187  */
 188 enum fail_dup_mod_reason {
 189         FAIL_DUP_MOD_BECOMING = 0,
 190         FAIL_DUP_MOD_LOAD,
 191 };
 192
 193 #ifdef CONFIG_MODULE_DEBUGFS
 194 extern struct dentry *mod_debugfs_root;
 195 #endif
 196
 197 #ifdef CONFIG_MODULE_STATS
 198
 199 #define mod_stat_add_long(count, var) atomic_long_add(count, var)
 200 #define mod_stat_inc(name) atomic_inc(name)
 201
 202 extern atomic_long_t total_mod_size;
 203 extern atomic_long_t total_text_size;
 204 extern atomic_long_t invalid_kread_bytes;
 205 extern atomic_long_t invalid_decompress_bytes;
 206
 207 extern atomic_t modcount;
 208 extern atomic_t failed_kreads;
 209 extern atomic_t failed_decompress;
 210 struct mod_fail_load {
 211         struct list_head list;
 212         char name[MODULE_NAME_LEN];
 213         atomic_long_t count;
 214         unsigned long dup_fail_mask;
 215 };
 216
 217 int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
 218 void mod_stat_bump_invalid(struct load_info *info, int flags);
 219 void mod_stat_bump_becoming(struct load_info *info, int flags);
 220
 221 #else
 222
 223 #define mod_stat_add_long(name, var)
 224 #define mod_stat_inc(name)
 225
 226 static inline int try_add_failed_module(const char *name,
 227                                         enum fail_dup_mod_reason reason)
 228 {
 229         return 0;
 230 }
 231
 232 static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
 233 {
 234 }
 235
 236 static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
 237 {
 238 }
 239
 240 #endif /* CONFIG_MODULE_STATS */
 241
 242 #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
 243 bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
 244 void kmod_dup_request_announce(char *module_name, int ret);
 245 #else
 246 static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
 247 {
 248         return false;
 249 }
 250
 251 static inline void kmod_dup_request_announce(char *module_name, int ret)
 252 {
 253 }
 254 #endif
 255
 256 #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
 257 struct mod_unload_taint {
 258         struct list_head list;
 259         char name[MODULE_NAME_LEN];
 260         unsigned long taints;
 261         u64 count;
 262 };
 263
 264 int try_add_tainted_module(struct module *mod);
 265 void print_unloaded_tainted_modules(void);
 266 #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
 267 static inline int try_add_tainted_module(struct module *mod)
 268 {
 269         return 0;
 270 }
 271
 272 static inline void print_unloaded_tainted_modules(void)
 273 {
 274 }
 275 #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
 276
 277 #ifdef CONFIG_MODULE_DECOMPRESS
 278 int module_decompress(struct load_info *info, const void *buf, size_t size);
 279 void module_decompress_cleanup(struct load_info *info);
 280 #else
 281 static inline int module_decompress(struct load_info *info,
 282                                     const void *buf, size_t size)
 283 {
 284         return -EOPNOTSUPP;
 285 }
 286
 287 static inline void module_decompress_cleanup(struct load_info *info)
 288 {
 289 }
 290 #endif
 291
 292 struct mod_tree_root {
 293 #ifdef CONFIG_MODULES_TREE_LOOKUP
 294         struct latch_tree_root root;
 295 #endif
 296         unsigned long addr_min;
 297         unsigned long addr_max;
 298 #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
 299         unsigned long data_addr_min;
 300         unsigned long data_addr_max;
 301 #endif
 302 };
 303
 304 extern struct mod_tree_root mod_tree;
 305
 306 #ifdef CONFIG_MODULES_TREE_LOOKUP
 307 void mod_tree_insert(struct module *mod);
 308 void mod_tree_remove_init(struct module *mod);
 309 void mod_tree_remove(struct module *mod);
 310 struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
 311 #else /* !CONFIG_MODULES_TREE_LOOKUP */
 312
 313 static inline void mod_tree_insert(struct module *mod) { }
 314 static inline void mod_tree_remove_init(struct module *mod) { }
 315 static inline void mod_tree_remove(struct module *mod) { }
 316 static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
 317 {
 318         struct module *mod;
 319
 320         list_for_each_entry_rcu(mod, &modules, list,
 321                                 lockdep_is_held(&module_mutex)) {
 322                 if (within_module(addr, mod))
 323                         return mod;
 324         }
 325
 326         return NULL;
 327 }
 328 #endif /* CONFIG_MODULES_TREE_LOOKUP */
 329
 330 int module_enable_rodata_ro(const struct module *mod, bool after_init);
 331 int module_enable_data_nx(const struct module *mod);
 332 int module_enable_text_rox(const struct module *mod);
 333 int module_enforce_rwx_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
 334                                 char *secstrings, struct module *mod);
 335
 336 #ifdef CONFIG_MODULE_SIG
 337 int module_sig_check(struct load_info *info, int flags);
 338 #else /* !CONFIG_MODULE_SIG */
 339 static inline int module_sig_check(struct load_info *info, int flags)
 340 {
 341         return 0;
 342 }
 343 #endif /* !CONFIG_MODULE_SIG */
 344
 345 #ifdef CONFIG_DEBUG_KMEMLEAK
 346 void kmemleak_load_module(const struct module *mod, const struct load_info *info);
 347 #else /* !CONFIG_DEBUG_KMEMLEAK */
 348 static inline void kmemleak_load_module(const struct module *mod,
 349                                         const struct load_info *info) { }
 350 #endif /* CONFIG_DEBUG_KMEMLEAK */
 351
 352 #ifdef CONFIG_KALLSYMS
 353 void init_build_id(struct module *mod, const struct load_info *info);
 354 void layout_symtab(struct module *mod, struct load_info *info);
 355 void add_kallsyms(struct module *mod, const struct load_info *info);
 356
 357 static inline bool sect_empty(const Elf_Shdr *sect)
 358 {
 359         return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
 360 }
 361 #else /* !CONFIG_KALLSYMS */
 362 static inline void init_build_id(struct module *mod, const struct load_info *info) { }
 363 static inline void layout_symtab(struct module *mod, struct load_info *info) { }
 364 static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
 365 #endif /* CONFIG_KALLSYMS */
 366
 367 #ifdef CONFIG_SYSFS
 368 int mod_sysfs_setup(struct module *mod, const struct load_info *info,
 369                     struct kernel_param *kparam, unsigned int num_params);
 370 void mod_sysfs_teardown(struct module *mod);
 371 void init_param_lock(struct module *mod);
 372 #else /* !CONFIG_SYSFS */
 373 static inline int mod_sysfs_setup(struct module *mod,
 374                                   const struct load_info *info,
 375                                   struct kernel_param *kparam,
 376                                   unsigned int num_params)
 377 {
 378         return 0;
 379 }
 380
 381 static inline void mod_sysfs_teardown(struct module *mod) { }
 382 static inline void init_param_lock(struct module *mod) { }
 383 #endif /* CONFIG_SYSFS */
 384
 385 #ifdef CONFIG_MODVERSIONS
 386 int check_version(const struct load_info *info,
 387                   const char *symname, struct module *mod, const s32 *crc);
 388 void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
 389                    struct kernel_symbol *ks, struct tracepoint * const *tp);
 390 int check_modstruct_version(const struct load_info *info, struct module *mod);
 391 int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
 392 #else /* !CONFIG_MODVERSIONS */
 393 static inline int check_version(const struct load_info *info,
 394                                 const char *symname,
 395                                 struct module *mod,
 396                                 const s32 *crc)
 397 {
 398         return 1;
 399 }
 400
 401 static inline int check_modstruct_version(const struct load_info *info,
 402                                           struct module *mod)
 403 {
 404         return 1;
 405 }
 406
 407 static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
 408 {
 409         return strcmp(amagic, bmagic) == 0;
 410 }
 411 #endif /* CONFIG_MODVERSIONS */