1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Procedures for maintaining information about logical memory blocks.
5 * Peter Bergner, IBM Corp. June 2001.
6 * Copyright (C) 2001 Peter Bergner.
9 #include <linux/kernel.h>
10 #include <linux/slab.h>
11 #include <linux/init.h>
12 #include <linux/bitops.h>
13 #include <linux/poison.h>
14 #include <linux/pfn.h>
15 #include <linux/debugfs.h>
16 #include <linux/kmemleak.h>
17 #include <linux/seq_file.h>
18 #include <linux/memblock.h>
20 #include <asm/sections.h>
25 #define INIT_MEMBLOCK_REGIONS 128
26 #define INIT_PHYSMEM_REGIONS 4
28 #ifndef INIT_MEMBLOCK_RESERVED_REGIONS
29 # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
33 * DOC: memblock overview
35 * Memblock is a method of managing memory regions during the early
36 * boot period when the usual kernel memory allocators are not up and
39 * Memblock views the system memory as collections of contiguous
40 * regions. There are several types of these collections:
42 * * ``memory`` - describes the physical memory available to the
43 * kernel; this may differ from the actual physical memory installed
44 * in the system, for instance when the memory is restricted with
45 * ``mem=`` command line parameter
46 * * ``reserved`` - describes the regions that were allocated
47 * * ``physmap`` - describes the actual physical memory regardless of
48 * the possible restrictions; the ``physmap`` type is only available
49 * on some architectures.
51 * Each region is represented by :c:type:`struct memblock_region` that
52 * defines the region extents, its attributes and NUMA node id on NUMA
53 * systems. Every memory type is described by the :c:type:`struct
54 * memblock_type` which contains an array of memory regions along with
55 * the allocator metadata. The memory types are nicely wrapped with
56 * :c:type:`struct memblock`. This structure is statically initialzed
57 * at build time. The region arrays for the "memory" and "reserved"
58 * types are initially sized to %INIT_MEMBLOCK_REGIONS and for the
59 * "physmap" type to %INIT_PHYSMEM_REGIONS.
60 * The :c:func:`memblock_allow_resize` enables automatic resizing of
61 * the region arrays during addition of new regions. This feature
62 * should be used with care so that memory allocated for the region
63 * array will not overlap with areas that should be reserved, for
66 * The early architecture setup should tell memblock what the physical
67 * memory layout is by using :c:func:`memblock_add` or
68 * :c:func:`memblock_add_node` functions. The first function does not
69 * assign the region to a NUMA node and it is appropriate for UMA
70 * systems. Yet, it is possible to use it on NUMA systems as well and
71 * assign the region to a NUMA node later in the setup process using
72 * :c:func:`memblock_set_node`. The :c:func:`memblock_add_node`
73 * performs such an assignment directly.
75 * Once memblock is setup the memory can be allocated using one of the
78 * * :c:func:`memblock_phys_alloc*` - these functions return the
79 * **physical** address of the allocated memory
80 * * :c:func:`memblock_alloc*` - these functions return the **virtual**
81 * address of the allocated memory.
83 * Note, that both API variants use implict assumptions about allowed
84 * memory ranges and the fallback methods. Consult the documentation
85 * of :c:func:`memblock_alloc_internal` and
86 * :c:func:`memblock_alloc_range_nid` functions for more elaboarte
89 * As the system boot progresses, the architecture specific
90 * :c:func:`mem_init` function frees all the memory to the buddy page
93 * Unless an architecure enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
94 * memblock data structures will be discarded after the system
95 * initialization compltes.
98 #ifndef CONFIG_NEED_MULTIPLE_NODES
99 struct pglist_data __refdata contig_page_data
;
100 EXPORT_SYMBOL(contig_page_data
);
103 unsigned long max_low_pfn
;
104 unsigned long min_low_pfn
;
105 unsigned long max_pfn
;
106 unsigned long long max_possible_pfn
;
108 static struct memblock_region memblock_memory_init_regions
[INIT_MEMBLOCK_REGIONS
] __initdata_memblock
;
109 static struct memblock_region memblock_reserved_init_regions
[INIT_MEMBLOCK_RESERVED_REGIONS
] __initdata_memblock
;
110 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
111 static struct memblock_region memblock_physmem_init_regions
[INIT_PHYSMEM_REGIONS
] __initdata_memblock
;
114 struct memblock memblock __initdata_memblock
= {
115 .memory
.regions
= memblock_memory_init_regions
,
116 .memory
.cnt
= 1, /* empty dummy entry */
117 .memory
.max
= INIT_MEMBLOCK_REGIONS
,
118 .memory
.name
= "memory",
120 .reserved
.regions
= memblock_reserved_init_regions
,
121 .reserved
.cnt
= 1, /* empty dummy entry */
122 .reserved
.max
= INIT_MEMBLOCK_RESERVED_REGIONS
,
123 .reserved
.name
= "reserved",
125 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
126 .physmem
.regions
= memblock_physmem_init_regions
,
127 .physmem
.cnt
= 1, /* empty dummy entry */
128 .physmem
.max
= INIT_PHYSMEM_REGIONS
,
129 .physmem
.name
= "physmem",
133 .current_limit
= MEMBLOCK_ALLOC_ANYWHERE
,
136 int memblock_debug __initdata_memblock
;
137 static bool system_has_some_mirror __initdata_memblock
= false;
138 static int memblock_can_resize __initdata_memblock
;
139 static int memblock_memory_in_slab __initdata_memblock
= 0;
140 static int memblock_reserved_in_slab __initdata_memblock
= 0;
142 static enum memblock_flags __init_memblock
choose_memblock_flags(void)
144 return system_has_some_mirror
? MEMBLOCK_MIRROR
: MEMBLOCK_NONE
;
147 /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
148 static inline phys_addr_t
memblock_cap_size(phys_addr_t base
, phys_addr_t
*size
)
150 return *size
= min(*size
, PHYS_ADDR_MAX
- base
);
154 * Address comparison utilities
156 static unsigned long __init_memblock
memblock_addrs_overlap(phys_addr_t base1
, phys_addr_t size1
,
157 phys_addr_t base2
, phys_addr_t size2
)
159 return ((base1
< (base2
+ size2
)) && (base2
< (base1
+ size1
)));
162 bool __init_memblock
memblock_overlaps_region(struct memblock_type
*type
,
163 phys_addr_t base
, phys_addr_t size
)
167 for (i
= 0; i
< type
->cnt
; i
++)
168 if (memblock_addrs_overlap(base
, size
, type
->regions
[i
].base
,
169 type
->regions
[i
].size
))
171 return i
< type
->cnt
;
175 * __memblock_find_range_bottom_up - find free area utility in bottom-up
176 * @start: start of candidate range
177 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
178 * %MEMBLOCK_ALLOC_ACCESSIBLE
179 * @size: size of free area to find
180 * @align: alignment of free area to find
181 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
182 * @flags: pick from blocks based on memory attributes
184 * Utility called from memblock_find_in_range_node(), find free area bottom-up.
187 * Found address on success, 0 on failure.
189 static phys_addr_t __init_memblock
190 __memblock_find_range_bottom_up(phys_addr_t start
, phys_addr_t end
,
191 phys_addr_t size
, phys_addr_t align
, int nid
,
192 enum memblock_flags flags
)
194 phys_addr_t this_start
, this_end
, cand
;
197 for_each_free_mem_range(i
, nid
, flags
, &this_start
, &this_end
, NULL
) {
198 this_start
= clamp(this_start
, start
, end
);
199 this_end
= clamp(this_end
, start
, end
);
201 cand
= round_up(this_start
, align
);
202 if (cand
< this_end
&& this_end
- cand
>= size
)
210 * __memblock_find_range_top_down - find free area utility, in top-down
211 * @start: start of candidate range
212 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
213 * %MEMBLOCK_ALLOC_ACCESSIBLE
214 * @size: size of free area to find
215 * @align: alignment of free area to find
216 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
217 * @flags: pick from blocks based on memory attributes
219 * Utility called from memblock_find_in_range_node(), find free area top-down.
222 * Found address on success, 0 on failure.
224 static phys_addr_t __init_memblock
225 __memblock_find_range_top_down(phys_addr_t start
, phys_addr_t end
,
226 phys_addr_t size
, phys_addr_t align
, int nid
,
227 enum memblock_flags flags
)
229 phys_addr_t this_start
, this_end
, cand
;
232 for_each_free_mem_range_reverse(i
, nid
, flags
, &this_start
, &this_end
,
234 this_start
= clamp(this_start
, start
, end
);
235 this_end
= clamp(this_end
, start
, end
);
240 cand
= round_down(this_end
- size
, align
);
241 if (cand
>= this_start
)
249 * memblock_find_in_range_node - find free area in given range and node
250 * @size: size of free area to find
251 * @align: alignment of free area to find
252 * @start: start of candidate range
253 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
254 * %MEMBLOCK_ALLOC_ACCESSIBLE
255 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
256 * @flags: pick from blocks based on memory attributes
258 * Find @size free area aligned to @align in the specified range and node.
260 * When allocation direction is bottom-up, the @start should be greater
261 * than the end of the kernel image. Otherwise, it will be trimmed. The
262 * reason is that we want the bottom-up allocation just near the kernel
263 * image so it is highly likely that the allocated memory and the kernel
264 * will reside in the same node.
266 * If bottom-up allocation failed, will try to allocate memory top-down.
269 * Found address on success, 0 on failure.
271 static phys_addr_t __init_memblock
memblock_find_in_range_node(phys_addr_t size
,
272 phys_addr_t align
, phys_addr_t start
,
273 phys_addr_t end
, int nid
,
274 enum memblock_flags flags
)
276 phys_addr_t kernel_end
, ret
;
279 if (end
== MEMBLOCK_ALLOC_ACCESSIBLE
||
280 end
== MEMBLOCK_ALLOC_KASAN
)
281 end
= memblock
.current_limit
;
283 /* avoid allocating the first page */
284 start
= max_t(phys_addr_t
, start
, PAGE_SIZE
);
285 end
= max(start
, end
);
286 kernel_end
= __pa_symbol(_end
);
289 * try bottom-up allocation only when bottom-up mode
290 * is set and @end is above the kernel image.
292 if (memblock_bottom_up() && end
> kernel_end
) {
293 phys_addr_t bottom_up_start
;
295 /* make sure we will allocate above the kernel */
296 bottom_up_start
= max(start
, kernel_end
);
298 /* ok, try bottom-up allocation first */
299 ret
= __memblock_find_range_bottom_up(bottom_up_start
, end
,
300 size
, align
, nid
, flags
);
305 * we always limit bottom-up allocation above the kernel,
306 * but top-down allocation doesn't have the limit, so
307 * retrying top-down allocation may succeed when bottom-up
310 * bottom-up allocation is expected to be fail very rarely,
311 * so we use WARN_ONCE() here to see the stack trace if
314 WARN_ONCE(IS_ENABLED(CONFIG_MEMORY_HOTREMOVE
),
315 "memblock: bottom-up allocation failed, memory hotremove may be affected\n");
318 return __memblock_find_range_top_down(start
, end
, size
, align
, nid
,
323 * memblock_find_in_range - find free area in given range
324 * @start: start of candidate range
325 * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
326 * %MEMBLOCK_ALLOC_ACCESSIBLE
327 * @size: size of free area to find
328 * @align: alignment of free area to find
330 * Find @size free area aligned to @align in the specified range.
333 * Found address on success, 0 on failure.
335 phys_addr_t __init_memblock
memblock_find_in_range(phys_addr_t start
,
336 phys_addr_t end
, phys_addr_t size
,
340 enum memblock_flags flags
= choose_memblock_flags();
343 ret
= memblock_find_in_range_node(size
, align
, start
, end
,
344 NUMA_NO_NODE
, flags
);
346 if (!ret
&& (flags
& MEMBLOCK_MIRROR
)) {
347 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
349 flags
&= ~MEMBLOCK_MIRROR
;
356 static void __init_memblock
memblock_remove_region(struct memblock_type
*type
, unsigned long r
)
358 type
->total_size
-= type
->regions
[r
].size
;
359 memmove(&type
->regions
[r
], &type
->regions
[r
+ 1],
360 (type
->cnt
- (r
+ 1)) * sizeof(type
->regions
[r
]));
363 /* Special case for empty arrays */
364 if (type
->cnt
== 0) {
365 WARN_ON(type
->total_size
!= 0);
367 type
->regions
[0].base
= 0;
368 type
->regions
[0].size
= 0;
369 type
->regions
[0].flags
= 0;
370 memblock_set_region_node(&type
->regions
[0], MAX_NUMNODES
);
374 #ifndef CONFIG_ARCH_KEEP_MEMBLOCK
376 * memblock_discard - discard memory and reserved arrays if they were allocated
378 void __init
memblock_discard(void)
380 phys_addr_t addr
, size
;
382 if (memblock
.reserved
.regions
!= memblock_reserved_init_regions
) {
383 addr
= __pa(memblock
.reserved
.regions
);
384 size
= PAGE_ALIGN(sizeof(struct memblock_region
) *
385 memblock
.reserved
.max
);
386 __memblock_free_late(addr
, size
);
389 if (memblock
.memory
.regions
!= memblock_memory_init_regions
) {
390 addr
= __pa(memblock
.memory
.regions
);
391 size
= PAGE_ALIGN(sizeof(struct memblock_region
) *
392 memblock
.memory
.max
);
393 __memblock_free_late(addr
, size
);
399 * memblock_double_array - double the size of the memblock regions array
400 * @type: memblock type of the regions array being doubled
401 * @new_area_start: starting address of memory range to avoid overlap with
402 * @new_area_size: size of memory range to avoid overlap with
404 * Double the size of the @type regions array. If memblock is being used to
405 * allocate memory for a new reserved regions array and there is a previously
406 * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
407 * waiting to be reserved, ensure the memory used by the new array does
411 * 0 on success, -1 on failure.
413 static int __init_memblock
memblock_double_array(struct memblock_type
*type
,
414 phys_addr_t new_area_start
,
415 phys_addr_t new_area_size
)
417 struct memblock_region
*new_array
, *old_array
;
418 phys_addr_t old_alloc_size
, new_alloc_size
;
419 phys_addr_t old_size
, new_size
, addr
, new_end
;
420 int use_slab
= slab_is_available();
423 /* We don't allow resizing until we know about the reserved regions
424 * of memory that aren't suitable for allocation
426 if (!memblock_can_resize
)
429 /* Calculate new doubled size */
430 old_size
= type
->max
* sizeof(struct memblock_region
);
431 new_size
= old_size
<< 1;
433 * We need to allocated new one align to PAGE_SIZE,
434 * so we can free them completely later.
436 old_alloc_size
= PAGE_ALIGN(old_size
);
437 new_alloc_size
= PAGE_ALIGN(new_size
);
439 /* Retrieve the slab flag */
440 if (type
== &memblock
.memory
)
441 in_slab
= &memblock_memory_in_slab
;
443 in_slab
= &memblock_reserved_in_slab
;
445 /* Try to find some space for it */
447 new_array
= kmalloc(new_size
, GFP_KERNEL
);
448 addr
= new_array
? __pa(new_array
) : 0;
450 /* only exclude range when trying to double reserved.regions */
451 if (type
!= &memblock
.reserved
)
452 new_area_start
= new_area_size
= 0;
454 addr
= memblock_find_in_range(new_area_start
+ new_area_size
,
455 memblock
.current_limit
,
456 new_alloc_size
, PAGE_SIZE
);
457 if (!addr
&& new_area_size
)
458 addr
= memblock_find_in_range(0,
459 min(new_area_start
, memblock
.current_limit
),
460 new_alloc_size
, PAGE_SIZE
);
462 new_array
= addr
? __va(addr
) : NULL
;
465 pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
466 type
->name
, type
->max
, type
->max
* 2);
470 new_end
= addr
+ new_size
- 1;
471 memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
472 type
->name
, type
->max
* 2, &addr
, &new_end
);
475 * Found space, we now need to move the array over before we add the
476 * reserved region since it may be our reserved array itself that is
479 memcpy(new_array
, type
->regions
, old_size
);
480 memset(new_array
+ type
->max
, 0, old_size
);
481 old_array
= type
->regions
;
482 type
->regions
= new_array
;
485 /* Free old array. We needn't free it if the array is the static one */
488 else if (old_array
!= memblock_memory_init_regions
&&
489 old_array
!= memblock_reserved_init_regions
)
490 memblock_free(__pa(old_array
), old_alloc_size
);
493 * Reserve the new array if that comes from the memblock. Otherwise, we
497 BUG_ON(memblock_reserve(addr
, new_alloc_size
));
499 /* Update slab flag */
506 * memblock_merge_regions - merge neighboring compatible regions
507 * @type: memblock type to scan
509 * Scan @type and merge neighboring compatible regions.
511 static void __init_memblock
memblock_merge_regions(struct memblock_type
*type
)
515 /* cnt never goes below 1 */
516 while (i
< type
->cnt
- 1) {
517 struct memblock_region
*this = &type
->regions
[i
];
518 struct memblock_region
*next
= &type
->regions
[i
+ 1];
520 if (this->base
+ this->size
!= next
->base
||
521 memblock_get_region_node(this) !=
522 memblock_get_region_node(next
) ||
523 this->flags
!= next
->flags
) {
524 BUG_ON(this->base
+ this->size
> next
->base
);
529 this->size
+= next
->size
;
530 /* move forward from next + 1, index of which is i + 2 */
531 memmove(next
, next
+ 1, (type
->cnt
- (i
+ 2)) * sizeof(*next
));
537 * memblock_insert_region - insert new memblock region
538 * @type: memblock type to insert into
539 * @idx: index for the insertion point
540 * @base: base address of the new region
541 * @size: size of the new region
542 * @nid: node id of the new region
543 * @flags: flags of the new region
545 * Insert new memblock region [@base, @base + @size) into @type at @idx.
546 * @type must already have extra room to accommodate the new region.
548 static void __init_memblock
memblock_insert_region(struct memblock_type
*type
,
549 int idx
, phys_addr_t base
,
552 enum memblock_flags flags
)
554 struct memblock_region
*rgn
= &type
->regions
[idx
];
556 BUG_ON(type
->cnt
>= type
->max
);
557 memmove(rgn
+ 1, rgn
, (type
->cnt
- idx
) * sizeof(*rgn
));
561 memblock_set_region_node(rgn
, nid
);
563 type
->total_size
+= size
;
567 * memblock_add_range - add new memblock region
568 * @type: memblock type to add new region into
569 * @base: base address of the new region
570 * @size: size of the new region
571 * @nid: nid of the new region
572 * @flags: flags of the new region
574 * Add new memblock region [@base, @base + @size) into @type. The new region
575 * is allowed to overlap with existing ones - overlaps don't affect already
576 * existing regions. @type is guaranteed to be minimal (all neighbouring
577 * compatible regions are merged) after the addition.
580 * 0 on success, -errno on failure.
582 int __init_memblock
memblock_add_range(struct memblock_type
*type
,
583 phys_addr_t base
, phys_addr_t size
,
584 int nid
, enum memblock_flags flags
)
587 phys_addr_t obase
= base
;
588 phys_addr_t end
= base
+ memblock_cap_size(base
, &size
);
590 struct memblock_region
*rgn
;
595 /* special case for empty array */
596 if (type
->regions
[0].size
== 0) {
597 WARN_ON(type
->cnt
!= 1 || type
->total_size
);
598 type
->regions
[0].base
= base
;
599 type
->regions
[0].size
= size
;
600 type
->regions
[0].flags
= flags
;
601 memblock_set_region_node(&type
->regions
[0], nid
);
602 type
->total_size
= size
;
607 * The following is executed twice. Once with %false @insert and
608 * then with %true. The first counts the number of regions needed
609 * to accommodate the new area. The second actually inserts them.
614 for_each_memblock_type(idx
, type
, rgn
) {
615 phys_addr_t rbase
= rgn
->base
;
616 phys_addr_t rend
= rbase
+ rgn
->size
;
623 * @rgn overlaps. If it separates the lower part of new
624 * area, insert that portion.
627 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
628 WARN_ON(nid
!= memblock_get_region_node(rgn
));
630 WARN_ON(flags
!= rgn
->flags
);
633 memblock_insert_region(type
, idx
++, base
,
637 /* area below @rend is dealt with, forget about it */
638 base
= min(rend
, end
);
641 /* insert the remaining portion */
645 memblock_insert_region(type
, idx
, base
, end
- base
,
653 * If this was the first round, resize array and repeat for actual
654 * insertions; otherwise, merge and return.
657 while (type
->cnt
+ nr_new
> type
->max
)
658 if (memblock_double_array(type
, obase
, size
) < 0)
663 memblock_merge_regions(type
);
669 * memblock_add_node - add new memblock region within a NUMA node
670 * @base: base address of the new region
671 * @size: size of the new region
672 * @nid: nid of the new region
674 * Add new memblock region [@base, @base + @size) to the "memory"
675 * type. See memblock_add_range() description for mode details
678 * 0 on success, -errno on failure.
680 int __init_memblock
memblock_add_node(phys_addr_t base
, phys_addr_t size
,
683 return memblock_add_range(&memblock
.memory
, base
, size
, nid
, 0);
687 * memblock_add - add new memblock region
688 * @base: base address of the new region
689 * @size: size of the new region
691 * Add new memblock region [@base, @base + @size) to the "memory"
692 * type. See memblock_add_range() description for mode details
695 * 0 on success, -errno on failure.
697 int __init_memblock
memblock_add(phys_addr_t base
, phys_addr_t size
)
699 phys_addr_t end
= base
+ size
- 1;
701 memblock_dbg("memblock_add: [%pa-%pa] %pS\n",
702 &base
, &end
, (void *)_RET_IP_
);
704 return memblock_add_range(&memblock
.memory
, base
, size
, MAX_NUMNODES
, 0);
708 * memblock_isolate_range - isolate given range into disjoint memblocks
709 * @type: memblock type to isolate range for
710 * @base: base of range to isolate
711 * @size: size of range to isolate
712 * @start_rgn: out parameter for the start of isolated region
713 * @end_rgn: out parameter for the end of isolated region
715 * Walk @type and ensure that regions don't cross the boundaries defined by
716 * [@base, @base + @size). Crossing regions are split at the boundaries,
717 * which may create at most two more regions. The index of the first
718 * region inside the range is returned in *@start_rgn and end in *@end_rgn.
721 * 0 on success, -errno on failure.
723 static int __init_memblock
memblock_isolate_range(struct memblock_type
*type
,
724 phys_addr_t base
, phys_addr_t size
,
725 int *start_rgn
, int *end_rgn
)
727 phys_addr_t end
= base
+ memblock_cap_size(base
, &size
);
729 struct memblock_region
*rgn
;
731 *start_rgn
= *end_rgn
= 0;
736 /* we'll create at most two more regions */
737 while (type
->cnt
+ 2 > type
->max
)
738 if (memblock_double_array(type
, base
, size
) < 0)
741 for_each_memblock_type(idx
, type
, rgn
) {
742 phys_addr_t rbase
= rgn
->base
;
743 phys_addr_t rend
= rbase
+ rgn
->size
;
752 * @rgn intersects from below. Split and continue
753 * to process the next region - the new top half.
756 rgn
->size
-= base
- rbase
;
757 type
->total_size
-= base
- rbase
;
758 memblock_insert_region(type
, idx
, rbase
, base
- rbase
,
759 memblock_get_region_node(rgn
),
761 } else if (rend
> end
) {
763 * @rgn intersects from above. Split and redo the
764 * current region - the new bottom half.
767 rgn
->size
-= end
- rbase
;
768 type
->total_size
-= end
- rbase
;
769 memblock_insert_region(type
, idx
--, rbase
, end
- rbase
,
770 memblock_get_region_node(rgn
),
773 /* @rgn is fully contained, record it */
783 static int __init_memblock
memblock_remove_range(struct memblock_type
*type
,
784 phys_addr_t base
, phys_addr_t size
)
786 int start_rgn
, end_rgn
;
789 ret
= memblock_isolate_range(type
, base
, size
, &start_rgn
, &end_rgn
);
793 for (i
= end_rgn
- 1; i
>= start_rgn
; i
--)
794 memblock_remove_region(type
, i
);
798 int __init_memblock
memblock_remove(phys_addr_t base
, phys_addr_t size
)
800 phys_addr_t end
= base
+ size
- 1;
802 memblock_dbg("memblock_remove: [%pa-%pa] %pS\n",
803 &base
, &end
, (void *)_RET_IP_
);
805 return memblock_remove_range(&memblock
.memory
, base
, size
);
809 * memblock_free - free boot memory block
810 * @base: phys starting address of the boot memory block
811 * @size: size of the boot memory block in bytes
813 * Free boot memory block previously allocated by memblock_alloc_xx() API.
814 * The freeing memory will not be released to the buddy allocator.
816 int __init_memblock
memblock_free(phys_addr_t base
, phys_addr_t size
)
818 phys_addr_t end
= base
+ size
- 1;
820 memblock_dbg(" memblock_free: [%pa-%pa] %pS\n",
821 &base
, &end
, (void *)_RET_IP_
);
823 kmemleak_free_part_phys(base
, size
);
824 return memblock_remove_range(&memblock
.reserved
, base
, size
);
827 int __init_memblock
memblock_reserve(phys_addr_t base
, phys_addr_t size
)
829 phys_addr_t end
= base
+ size
- 1;
831 memblock_dbg("memblock_reserve: [%pa-%pa] %pS\n",
832 &base
, &end
, (void *)_RET_IP_
);
834 return memblock_add_range(&memblock
.reserved
, base
, size
, MAX_NUMNODES
, 0);
838 * memblock_setclr_flag - set or clear flag for a memory region
839 * @base: base address of the region
840 * @size: size of the region
841 * @set: set or clear the flag
842 * @flag: the flag to udpate
844 * This function isolates region [@base, @base + @size), and sets/clears flag
846 * Return: 0 on success, -errno on failure.
848 static int __init_memblock
memblock_setclr_flag(phys_addr_t base
,
849 phys_addr_t size
, int set
, int flag
)
851 struct memblock_type
*type
= &memblock
.memory
;
852 int i
, ret
, start_rgn
, end_rgn
;
854 ret
= memblock_isolate_range(type
, base
, size
, &start_rgn
, &end_rgn
);
858 for (i
= start_rgn
; i
< end_rgn
; i
++) {
859 struct memblock_region
*r
= &type
->regions
[i
];
867 memblock_merge_regions(type
);
872 * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
873 * @base: the base phys addr of the region
874 * @size: the size of the region
876 * Return: 0 on success, -errno on failure.
878 int __init_memblock
memblock_mark_hotplug(phys_addr_t base
, phys_addr_t size
)
880 return memblock_setclr_flag(base
, size
, 1, MEMBLOCK_HOTPLUG
);
884 * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
885 * @base: the base phys addr of the region
886 * @size: the size of the region
888 * Return: 0 on success, -errno on failure.
890 int __init_memblock
memblock_clear_hotplug(phys_addr_t base
, phys_addr_t size
)
892 return memblock_setclr_flag(base
, size
, 0, MEMBLOCK_HOTPLUG
);
896 * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
897 * @base: the base phys addr of the region
898 * @size: the size of the region
900 * Return: 0 on success, -errno on failure.
902 int __init_memblock
memblock_mark_mirror(phys_addr_t base
, phys_addr_t size
)
904 system_has_some_mirror
= true;
906 return memblock_setclr_flag(base
, size
, 1, MEMBLOCK_MIRROR
);
910 * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
911 * @base: the base phys addr of the region
912 * @size: the size of the region
914 * Return: 0 on success, -errno on failure.
916 int __init_memblock
memblock_mark_nomap(phys_addr_t base
, phys_addr_t size
)
918 return memblock_setclr_flag(base
, size
, 1, MEMBLOCK_NOMAP
);
922 * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
923 * @base: the base phys addr of the region
924 * @size: the size of the region
926 * Return: 0 on success, -errno on failure.
928 int __init_memblock
memblock_clear_nomap(phys_addr_t base
, phys_addr_t size
)
930 return memblock_setclr_flag(base
, size
, 0, MEMBLOCK_NOMAP
);
934 * __next_reserved_mem_region - next function for for_each_reserved_region()
935 * @idx: pointer to u64 loop variable
936 * @out_start: ptr to phys_addr_t for start address of the region, can be %NULL
937 * @out_end: ptr to phys_addr_t for end address of the region, can be %NULL
939 * Iterate over all reserved memory regions.
941 void __init_memblock
__next_reserved_mem_region(u64
*idx
,
942 phys_addr_t
*out_start
,
943 phys_addr_t
*out_end
)
945 struct memblock_type
*type
= &memblock
.reserved
;
947 if (*idx
< type
->cnt
) {
948 struct memblock_region
*r
= &type
->regions
[*idx
];
949 phys_addr_t base
= r
->base
;
950 phys_addr_t size
= r
->size
;
955 *out_end
= base
+ size
- 1;
961 /* signal end of iteration */
965 static bool should_skip_region(struct memblock_region
*m
, int nid
, int flags
)
967 int m_nid
= memblock_get_region_node(m
);
969 /* only memory regions are associated with nodes, check it */
970 if (nid
!= NUMA_NO_NODE
&& nid
!= m_nid
)
973 /* skip hotpluggable memory regions if needed */
974 if (movable_node_is_enabled() && memblock_is_hotpluggable(m
))
977 /* if we want mirror memory skip non-mirror memory regions */
978 if ((flags
& MEMBLOCK_MIRROR
) && !memblock_is_mirror(m
))
981 /* skip nomap memory unless we were asked for it explicitly */
982 if (!(flags
& MEMBLOCK_NOMAP
) && memblock_is_nomap(m
))
989 * __next_mem_range - next function for for_each_free_mem_range() etc.
990 * @idx: pointer to u64 loop variable
991 * @nid: node selector, %NUMA_NO_NODE for all nodes
992 * @flags: pick from blocks based on memory attributes
993 * @type_a: pointer to memblock_type from where the range is taken
994 * @type_b: pointer to memblock_type which excludes memory from being taken
995 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
996 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
997 * @out_nid: ptr to int for nid of the range, can be %NULL
999 * Find the first area from *@idx which matches @nid, fill the out
1000 * parameters, and update *@idx for the next iteration. The lower 32bit of
1001 * *@idx contains index into type_a and the upper 32bit indexes the
1002 * areas before each region in type_b. For example, if type_b regions
1003 * look like the following,
1005 * 0:[0-16), 1:[32-48), 2:[128-130)
1007 * The upper 32bit indexes the following regions.
1009 * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
1011 * As both region arrays are sorted, the function advances the two indices
1012 * in lockstep and returns each intersection.
1014 void __init_memblock
__next_mem_range(u64
*idx
, int nid
,
1015 enum memblock_flags flags
,
1016 struct memblock_type
*type_a
,
1017 struct memblock_type
*type_b
,
1018 phys_addr_t
*out_start
,
1019 phys_addr_t
*out_end
, int *out_nid
)
1021 int idx_a
= *idx
& 0xffffffff;
1022 int idx_b
= *idx
>> 32;
1024 if (WARN_ONCE(nid
== MAX_NUMNODES
,
1025 "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1028 for (; idx_a
< type_a
->cnt
; idx_a
++) {
1029 struct memblock_region
*m
= &type_a
->regions
[idx_a
];
1031 phys_addr_t m_start
= m
->base
;
1032 phys_addr_t m_end
= m
->base
+ m
->size
;
1033 int m_nid
= memblock_get_region_node(m
);
1035 if (should_skip_region(m
, nid
, flags
))
1040 *out_start
= m_start
;
1046 *idx
= (u32
)idx_a
| (u64
)idx_b
<< 32;
1050 /* scan areas before each reservation */
1051 for (; idx_b
< type_b
->cnt
+ 1; idx_b
++) {
1052 struct memblock_region
*r
;
1053 phys_addr_t r_start
;
1056 r
= &type_b
->regions
[idx_b
];
1057 r_start
= idx_b
? r
[-1].base
+ r
[-1].size
: 0;
1058 r_end
= idx_b
< type_b
->cnt
?
1059 r
->base
: PHYS_ADDR_MAX
;
1062 * if idx_b advanced past idx_a,
1063 * break out to advance idx_a
1065 if (r_start
>= m_end
)
1067 /* if the two regions intersect, we're done */
1068 if (m_start
< r_end
) {
1071 max(m_start
, r_start
);
1073 *out_end
= min(m_end
, r_end
);
1077 * The region which ends first is
1078 * advanced for the next iteration.
1084 *idx
= (u32
)idx_a
| (u64
)idx_b
<< 32;
1090 /* signal end of iteration */
1095 * __next_mem_range_rev - generic next function for for_each_*_range_rev()
1097 * @idx: pointer to u64 loop variable
1098 * @nid: node selector, %NUMA_NO_NODE for all nodes
1099 * @flags: pick from blocks based on memory attributes
1100 * @type_a: pointer to memblock_type from where the range is taken
1101 * @type_b: pointer to memblock_type which excludes memory from being taken
1102 * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
1103 * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
1104 * @out_nid: ptr to int for nid of the range, can be %NULL
1106 * Finds the next range from type_a which is not marked as unsuitable
1109 * Reverse of __next_mem_range().
1111 void __init_memblock
__next_mem_range_rev(u64
*idx
, int nid
,
1112 enum memblock_flags flags
,
1113 struct memblock_type
*type_a
,
1114 struct memblock_type
*type_b
,
1115 phys_addr_t
*out_start
,
1116 phys_addr_t
*out_end
, int *out_nid
)
1118 int idx_a
= *idx
& 0xffffffff;
1119 int idx_b
= *idx
>> 32;
1121 if (WARN_ONCE(nid
== MAX_NUMNODES
, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1124 if (*idx
== (u64
)ULLONG_MAX
) {
1125 idx_a
= type_a
->cnt
- 1;
1127 idx_b
= type_b
->cnt
;
1132 for (; idx_a
>= 0; idx_a
--) {
1133 struct memblock_region
*m
= &type_a
->regions
[idx_a
];
1135 phys_addr_t m_start
= m
->base
;
1136 phys_addr_t m_end
= m
->base
+ m
->size
;
1137 int m_nid
= memblock_get_region_node(m
);
1139 if (should_skip_region(m
, nid
, flags
))
1144 *out_start
= m_start
;
1150 *idx
= (u32
)idx_a
| (u64
)idx_b
<< 32;
1154 /* scan areas before each reservation */
1155 for (; idx_b
>= 0; idx_b
--) {
1156 struct memblock_region
*r
;
1157 phys_addr_t r_start
;
1160 r
= &type_b
->regions
[idx_b
];
1161 r_start
= idx_b
? r
[-1].base
+ r
[-1].size
: 0;
1162 r_end
= idx_b
< type_b
->cnt
?
1163 r
->base
: PHYS_ADDR_MAX
;
1165 * if idx_b advanced past idx_a,
1166 * break out to advance idx_a
1169 if (r_end
<= m_start
)
1171 /* if the two regions intersect, we're done */
1172 if (m_end
> r_start
) {
1174 *out_start
= max(m_start
, r_start
);
1176 *out_end
= min(m_end
, r_end
);
1179 if (m_start
>= r_start
)
1183 *idx
= (u32
)idx_a
| (u64
)idx_b
<< 32;
1188 /* signal end of iteration */
1192 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1194 * Common iterator interface used to define for_each_mem_pfn_range().
1196 void __init_memblock
__next_mem_pfn_range(int *idx
, int nid
,
1197 unsigned long *out_start_pfn
,
1198 unsigned long *out_end_pfn
, int *out_nid
)
1200 struct memblock_type
*type
= &memblock
.memory
;
1201 struct memblock_region
*r
;
1203 while (++*idx
< type
->cnt
) {
1204 r
= &type
->regions
[*idx
];
1206 if (PFN_UP(r
->base
) >= PFN_DOWN(r
->base
+ r
->size
))
1208 if (nid
== MAX_NUMNODES
|| nid
== r
->nid
)
1211 if (*idx
>= type
->cnt
) {
1217 *out_start_pfn
= PFN_UP(r
->base
);
1219 *out_end_pfn
= PFN_DOWN(r
->base
+ r
->size
);
1225 * memblock_set_node - set node ID on memblock regions
1226 * @base: base of area to set node ID for
1227 * @size: size of area to set node ID for
1228 * @type: memblock type to set node ID for
1229 * @nid: node ID to set
1231 * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
1232 * Regions which cross the area boundaries are split as necessary.
1235 * 0 on success, -errno on failure.
1237 int __init_memblock
memblock_set_node(phys_addr_t base
, phys_addr_t size
,
1238 struct memblock_type
*type
, int nid
)
1240 int start_rgn
, end_rgn
;
1243 ret
= memblock_isolate_range(type
, base
, size
, &start_rgn
, &end_rgn
);
1247 for (i
= start_rgn
; i
< end_rgn
; i
++)
1248 memblock_set_region_node(&type
->regions
[i
], nid
);
1250 memblock_merge_regions(type
);
1253 #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1254 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1256 * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
1258 * @idx: pointer to u64 loop variable
1259 * @zone: zone in which all of the memory blocks reside
1260 * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
1261 * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
1263 * This function is meant to be a zone/pfn specific wrapper for the
1264 * for_each_mem_range type iterators. Specifically they are used in the
1265 * deferred memory init routines and as such we were duplicating much of
1266 * this logic throughout the code. So instead of having it in multiple
1267 * locations it seemed like it would make more sense to centralize this to
1268 * one new iterator that does everything they need.
1270 void __init_memblock
1271 __next_mem_pfn_range_in_zone(u64
*idx
, struct zone
*zone
,
1272 unsigned long *out_spfn
, unsigned long *out_epfn
)
1274 int zone_nid
= zone_to_nid(zone
);
1275 phys_addr_t spa
, epa
;
1278 __next_mem_range(idx
, zone_nid
, MEMBLOCK_NONE
,
1279 &memblock
.memory
, &memblock
.reserved
,
1282 while (*idx
!= U64_MAX
) {
1283 unsigned long epfn
= PFN_DOWN(epa
);
1284 unsigned long spfn
= PFN_UP(spa
);
1287 * Verify the end is at least past the start of the zone and
1288 * that we have at least one PFN to initialize.
1290 if (zone
->zone_start_pfn
< epfn
&& spfn
< epfn
) {
1291 /* if we went too far just stop searching */
1292 if (zone_end_pfn(zone
) <= spfn
) {
1298 *out_spfn
= max(zone
->zone_start_pfn
, spfn
);
1300 *out_epfn
= min(zone_end_pfn(zone
), epfn
);
1305 __next_mem_range(idx
, zone_nid
, MEMBLOCK_NONE
,
1306 &memblock
.memory
, &memblock
.reserved
,
1310 /* signal end of iteration */
1312 *out_spfn
= ULONG_MAX
;
1317 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
1320 * memblock_alloc_range_nid - allocate boot memory block
1321 * @size: size of memory block to be allocated in bytes
1322 * @align: alignment of the region and block's size
1323 * @start: the lower bound of the memory region to allocate (phys address)
1324 * @end: the upper bound of the memory region to allocate (phys address)
1325 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1327 * The allocation is performed from memory region limited by
1328 * memblock.current_limit if @max_addr == %MEMBLOCK_ALLOC_ACCESSIBLE.
1330 * If the specified node can not hold the requested memory the
1331 * allocation falls back to any node in the system
1333 * For systems with memory mirroring, the allocation is attempted first
1334 * from the regions with mirroring enabled and then retried from any
1337 * In addition, function sets the min_count to 0 using kmemleak_alloc_phys for
1338 * allocated boot memory block, so that it is never reported as leaks.
1341 * Physical address of allocated memory block on success, %0 on failure.
1343 static phys_addr_t __init
memblock_alloc_range_nid(phys_addr_t size
,
1344 phys_addr_t align
, phys_addr_t start
,
1345 phys_addr_t end
, int nid
)
1347 enum memblock_flags flags
= choose_memblock_flags();
1350 if (WARN_ONCE(nid
== MAX_NUMNODES
, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
1354 /* Can't use WARNs this early in boot on powerpc */
1356 align
= SMP_CACHE_BYTES
;
1359 if (end
> memblock
.current_limit
)
1360 end
= memblock
.current_limit
;
1363 found
= memblock_find_in_range_node(size
, align
, start
, end
, nid
,
1365 if (found
&& !memblock_reserve(found
, size
))
1368 if (nid
!= NUMA_NO_NODE
) {
1369 found
= memblock_find_in_range_node(size
, align
, start
,
1372 if (found
&& !memblock_reserve(found
, size
))
1376 if (flags
& MEMBLOCK_MIRROR
) {
1377 flags
&= ~MEMBLOCK_MIRROR
;
1378 pr_warn("Could not allocate %pap bytes of mirrored memory\n",
1386 /* Skip kmemleak for kasan_init() due to high volume. */
1387 if (end
!= MEMBLOCK_ALLOC_KASAN
)
1389 * The min_count is set to 0 so that memblock allocated
1390 * blocks are never reported as leaks. This is because many
1391 * of these blocks are only referred via the physical
1392 * address which is not looked up by kmemleak.
1394 kmemleak_alloc_phys(found
, size
, 0, 0);
1400 * memblock_phys_alloc_range - allocate a memory block inside specified range
1401 * @size: size of memory block to be allocated in bytes
1402 * @align: alignment of the region and block's size
1403 * @start: the lower bound of the memory region to allocate (physical address)
1404 * @end: the upper bound of the memory region to allocate (physical address)
1406 * Allocate @size bytes in the between @start and @end.
1408 * Return: physical address of the allocated memory block on success,
1411 phys_addr_t __init
memblock_phys_alloc_range(phys_addr_t size
,
1416 return memblock_alloc_range_nid(size
, align
, start
, end
, NUMA_NO_NODE
);
1420 * memblock_phys_alloc_try_nid - allocate a memory block from specified MUMA node
1421 * @size: size of memory block to be allocated in bytes
1422 * @align: alignment of the region and block's size
1423 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1425 * Allocates memory block from the specified NUMA node. If the node
1426 * has no available memory, attempts to allocated from any node in the
1429 * Return: physical address of the allocated memory block on success,
1432 phys_addr_t __init
memblock_phys_alloc_try_nid(phys_addr_t size
, phys_addr_t align
, int nid
)
1434 return memblock_alloc_range_nid(size
, align
, 0,
1435 MEMBLOCK_ALLOC_ACCESSIBLE
, nid
);
1439 * memblock_alloc_internal - allocate boot memory block
1440 * @size: size of memory block to be allocated in bytes
1441 * @align: alignment of the region and block's size
1442 * @min_addr: the lower bound of the memory region to allocate (phys address)
1443 * @max_addr: the upper bound of the memory region to allocate (phys address)
1444 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1446 * Allocates memory block using memblock_alloc_range_nid() and
1447 * converts the returned physical address to virtual.
1449 * The @min_addr limit is dropped if it can not be satisfied and the allocation
1450 * will fall back to memory below @min_addr. Other constraints, such
1451 * as node and mirrored memory will be handled again in
1452 * memblock_alloc_range_nid().
1455 * Virtual address of allocated memory block on success, NULL on failure.
1457 static void * __init
memblock_alloc_internal(
1458 phys_addr_t size
, phys_addr_t align
,
1459 phys_addr_t min_addr
, phys_addr_t max_addr
,
1465 * Detect any accidental use of these APIs after slab is ready, as at
1466 * this moment memblock may be deinitialized already and its
1467 * internal data may be destroyed (after execution of memblock_free_all)
1469 if (WARN_ON_ONCE(slab_is_available()))
1470 return kzalloc_node(size
, GFP_NOWAIT
, nid
);
1472 alloc
= memblock_alloc_range_nid(size
, align
, min_addr
, max_addr
, nid
);
1474 /* retry allocation without lower limit */
1475 if (!alloc
&& min_addr
)
1476 alloc
= memblock_alloc_range_nid(size
, align
, 0, max_addr
, nid
);
1481 return phys_to_virt(alloc
);
1485 * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
1486 * memory and without panicking
1487 * @size: size of memory block to be allocated in bytes
1488 * @align: alignment of the region and block's size
1489 * @min_addr: the lower bound of the memory region from where the allocation
1490 * is preferred (phys address)
1491 * @max_addr: the upper bound of the memory region from where the allocation
1492 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1493 * allocate only from memory limited by memblock.current_limit value
1494 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1496 * Public function, provides additional debug information (including caller
1497 * info), if enabled. Does not zero allocated memory, does not panic if request
1498 * cannot be satisfied.
1501 * Virtual address of allocated memory block on success, NULL on failure.
1503 void * __init
memblock_alloc_try_nid_raw(
1504 phys_addr_t size
, phys_addr_t align
,
1505 phys_addr_t min_addr
, phys_addr_t max_addr
,
1510 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1511 __func__
, (u64
)size
, (u64
)align
, nid
, &min_addr
,
1512 &max_addr
, (void *)_RET_IP_
);
1514 ptr
= memblock_alloc_internal(size
, align
,
1515 min_addr
, max_addr
, nid
);
1516 if (ptr
&& size
> 0)
1517 page_init_poison(ptr
, size
);
1523 * memblock_alloc_try_nid - allocate boot memory block
1524 * @size: size of memory block to be allocated in bytes
1525 * @align: alignment of the region and block's size
1526 * @min_addr: the lower bound of the memory region from where the allocation
1527 * is preferred (phys address)
1528 * @max_addr: the upper bound of the memory region from where the allocation
1529 * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
1530 * allocate only from memory limited by memblock.current_limit value
1531 * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
1533 * Public function, provides additional debug information (including caller
1534 * info), if enabled. This function zeroes the allocated memory.
1537 * Virtual address of allocated memory block on success, NULL on failure.
1539 void * __init
memblock_alloc_try_nid(
1540 phys_addr_t size
, phys_addr_t align
,
1541 phys_addr_t min_addr
, phys_addr_t max_addr
,
1546 memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
1547 __func__
, (u64
)size
, (u64
)align
, nid
, &min_addr
,
1548 &max_addr
, (void *)_RET_IP_
);
1549 ptr
= memblock_alloc_internal(size
, align
,
1550 min_addr
, max_addr
, nid
);
1552 memset(ptr
, 0, size
);
1558 * __memblock_free_late - free pages directly to buddy allocator
1559 * @base: phys starting address of the boot memory block
1560 * @size: size of the boot memory block in bytes
1562 * This is only useful when the memblock allocator has already been torn
1563 * down, but we are still initializing the system. Pages are released directly
1564 * to the buddy allocator.
1566 void __init
__memblock_free_late(phys_addr_t base
, phys_addr_t size
)
1568 phys_addr_t cursor
, end
;
1570 end
= base
+ size
- 1;
1571 memblock_dbg("%s: [%pa-%pa] %pS\n",
1572 __func__
, &base
, &end
, (void *)_RET_IP_
);
1573 kmemleak_free_part_phys(base
, size
);
1574 cursor
= PFN_UP(base
);
1575 end
= PFN_DOWN(base
+ size
);
1577 for (; cursor
< end
; cursor
++) {
1578 memblock_free_pages(pfn_to_page(cursor
), cursor
, 0);
1579 totalram_pages_inc();
1584 * Remaining API functions
1587 phys_addr_t __init_memblock
memblock_phys_mem_size(void)
1589 return memblock
.memory
.total_size
;
1592 phys_addr_t __init_memblock
memblock_reserved_size(void)
1594 return memblock
.reserved
.total_size
;
1597 phys_addr_t __init
memblock_mem_size(unsigned long limit_pfn
)
1599 unsigned long pages
= 0;
1600 struct memblock_region
*r
;
1601 unsigned long start_pfn
, end_pfn
;
1603 for_each_memblock(memory
, r
) {
1604 start_pfn
= memblock_region_memory_base_pfn(r
);
1605 end_pfn
= memblock_region_memory_end_pfn(r
);
1606 start_pfn
= min_t(unsigned long, start_pfn
, limit_pfn
);
1607 end_pfn
= min_t(unsigned long, end_pfn
, limit_pfn
);
1608 pages
+= end_pfn
- start_pfn
;
1611 return PFN_PHYS(pages
);
1614 /* lowest address */
1615 phys_addr_t __init_memblock
memblock_start_of_DRAM(void)
1617 return memblock
.memory
.regions
[0].base
;
1620 phys_addr_t __init_memblock
memblock_end_of_DRAM(void)
1622 int idx
= memblock
.memory
.cnt
- 1;
1624 return (memblock
.memory
.regions
[idx
].base
+ memblock
.memory
.regions
[idx
].size
);
1627 static phys_addr_t __init_memblock
__find_max_addr(phys_addr_t limit
)
1629 phys_addr_t max_addr
= PHYS_ADDR_MAX
;
1630 struct memblock_region
*r
;
1633 * translate the memory @limit size into the max address within one of
1634 * the memory memblock regions, if the @limit exceeds the total size
1635 * of those regions, max_addr will keep original value PHYS_ADDR_MAX
1637 for_each_memblock(memory
, r
) {
1638 if (limit
<= r
->size
) {
1639 max_addr
= r
->base
+ limit
;
1648 void __init
memblock_enforce_memory_limit(phys_addr_t limit
)
1650 phys_addr_t max_addr
= PHYS_ADDR_MAX
;
1655 max_addr
= __find_max_addr(limit
);
1657 /* @limit exceeds the total size of the memory, do nothing */
1658 if (max_addr
== PHYS_ADDR_MAX
)
1661 /* truncate both memory and reserved regions */
1662 memblock_remove_range(&memblock
.memory
, max_addr
,
1664 memblock_remove_range(&memblock
.reserved
, max_addr
,
1668 void __init
memblock_cap_memory_range(phys_addr_t base
, phys_addr_t size
)
1670 int start_rgn
, end_rgn
;
1676 ret
= memblock_isolate_range(&memblock
.memory
, base
, size
,
1677 &start_rgn
, &end_rgn
);
1681 /* remove all the MAP regions */
1682 for (i
= memblock
.memory
.cnt
- 1; i
>= end_rgn
; i
--)
1683 if (!memblock_is_nomap(&memblock
.memory
.regions
[i
]))
1684 memblock_remove_region(&memblock
.memory
, i
);
1686 for (i
= start_rgn
- 1; i
>= 0; i
--)
1687 if (!memblock_is_nomap(&memblock
.memory
.regions
[i
]))
1688 memblock_remove_region(&memblock
.memory
, i
);
1690 /* truncate the reserved regions */
1691 memblock_remove_range(&memblock
.reserved
, 0, base
);
1692 memblock_remove_range(&memblock
.reserved
,
1693 base
+ size
, PHYS_ADDR_MAX
);
1696 void __init
memblock_mem_limit_remove_map(phys_addr_t limit
)
1698 phys_addr_t max_addr
;
1703 max_addr
= __find_max_addr(limit
);
1705 /* @limit exceeds the total size of the memory, do nothing */
1706 if (max_addr
== PHYS_ADDR_MAX
)
1709 memblock_cap_memory_range(0, max_addr
);
1712 static int __init_memblock
memblock_search(struct memblock_type
*type
, phys_addr_t addr
)
1714 unsigned int left
= 0, right
= type
->cnt
;
1717 unsigned int mid
= (right
+ left
) / 2;
1719 if (addr
< type
->regions
[mid
].base
)
1721 else if (addr
>= (type
->regions
[mid
].base
+
1722 type
->regions
[mid
].size
))
1726 } while (left
< right
);
1730 bool __init_memblock
memblock_is_reserved(phys_addr_t addr
)
1732 return memblock_search(&memblock
.reserved
, addr
) != -1;
1735 bool __init_memblock
memblock_is_memory(phys_addr_t addr
)
1737 return memblock_search(&memblock
.memory
, addr
) != -1;
1740 bool __init_memblock
memblock_is_map_memory(phys_addr_t addr
)
1742 int i
= memblock_search(&memblock
.memory
, addr
);
1746 return !memblock_is_nomap(&memblock
.memory
.regions
[i
]);
1749 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1750 int __init_memblock
memblock_search_pfn_nid(unsigned long pfn
,
1751 unsigned long *start_pfn
, unsigned long *end_pfn
)
1753 struct memblock_type
*type
= &memblock
.memory
;
1754 int mid
= memblock_search(type
, PFN_PHYS(pfn
));
1759 *start_pfn
= PFN_DOWN(type
->regions
[mid
].base
);
1760 *end_pfn
= PFN_DOWN(type
->regions
[mid
].base
+ type
->regions
[mid
].size
);
1762 return type
->regions
[mid
].nid
;
1767 * memblock_is_region_memory - check if a region is a subset of memory
1768 * @base: base of region to check
1769 * @size: size of region to check
1771 * Check if the region [@base, @base + @size) is a subset of a memory block.
1774 * 0 if false, non-zero if true
1776 bool __init_memblock
memblock_is_region_memory(phys_addr_t base
, phys_addr_t size
)
1778 int idx
= memblock_search(&memblock
.memory
, base
);
1779 phys_addr_t end
= base
+ memblock_cap_size(base
, &size
);
1783 return (memblock
.memory
.regions
[idx
].base
+
1784 memblock
.memory
.regions
[idx
].size
) >= end
;
1788 * memblock_is_region_reserved - check if a region intersects reserved memory
1789 * @base: base of region to check
1790 * @size: size of region to check
1792 * Check if the region [@base, @base + @size) intersects a reserved
1796 * True if they intersect, false if not.
1798 bool __init_memblock
memblock_is_region_reserved(phys_addr_t base
, phys_addr_t size
)
1800 memblock_cap_size(base
, &size
);
1801 return memblock_overlaps_region(&memblock
.reserved
, base
, size
);
1804 void __init_memblock
memblock_trim_memory(phys_addr_t align
)
1806 phys_addr_t start
, end
, orig_start
, orig_end
;
1807 struct memblock_region
*r
;
1809 for_each_memblock(memory
, r
) {
1810 orig_start
= r
->base
;
1811 orig_end
= r
->base
+ r
->size
;
1812 start
= round_up(orig_start
, align
);
1813 end
= round_down(orig_end
, align
);
1815 if (start
== orig_start
&& end
== orig_end
)
1820 r
->size
= end
- start
;
1822 memblock_remove_region(&memblock
.memory
,
1823 r
- memblock
.memory
.regions
);
1829 void __init_memblock
memblock_set_current_limit(phys_addr_t limit
)
1831 memblock
.current_limit
= limit
;
1834 phys_addr_t __init_memblock
memblock_get_current_limit(void)
1836 return memblock
.current_limit
;
1839 static void __init_memblock
memblock_dump(struct memblock_type
*type
)
1841 phys_addr_t base
, end
, size
;
1842 enum memblock_flags flags
;
1844 struct memblock_region
*rgn
;
1846 pr_info(" %s.cnt = 0x%lx\n", type
->name
, type
->cnt
);
1848 for_each_memblock_type(idx
, type
, rgn
) {
1849 char nid_buf
[32] = "";
1853 end
= base
+ size
- 1;
1855 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1856 if (memblock_get_region_node(rgn
) != MAX_NUMNODES
)
1857 snprintf(nid_buf
, sizeof(nid_buf
), " on node %d",
1858 memblock_get_region_node(rgn
));
1860 pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
1861 type
->name
, idx
, &base
, &end
, &size
, nid_buf
, flags
);
1865 void __init_memblock
__memblock_dump_all(void)
1867 pr_info("MEMBLOCK configuration:\n");
1868 pr_info(" memory size = %pa reserved size = %pa\n",
1869 &memblock
.memory
.total_size
,
1870 &memblock
.reserved
.total_size
);
1872 memblock_dump(&memblock
.memory
);
1873 memblock_dump(&memblock
.reserved
);
1874 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
1875 memblock_dump(&memblock
.physmem
);
1879 void __init
memblock_allow_resize(void)
1881 memblock_can_resize
= 1;
1884 static int __init
early_memblock(char *p
)
1886 if (p
&& strstr(p
, "debug"))
1890 early_param("memblock", early_memblock
);
1892 static void __init
__free_pages_memory(unsigned long start
, unsigned long end
)
1896 while (start
< end
) {
1897 order
= min(MAX_ORDER
- 1UL, __ffs(start
));
1899 while (start
+ (1UL << order
) > end
)
1902 memblock_free_pages(pfn_to_page(start
), start
, order
);
1904 start
+= (1UL << order
);
1908 static unsigned long __init
__free_memory_core(phys_addr_t start
,
1911 unsigned long start_pfn
= PFN_UP(start
);
1912 unsigned long end_pfn
= min_t(unsigned long,
1913 PFN_DOWN(end
), max_low_pfn
);
1915 if (start_pfn
>= end_pfn
)
1918 __free_pages_memory(start_pfn
, end_pfn
);
1920 return end_pfn
- start_pfn
;
1923 static unsigned long __init
free_low_memory_core_early(void)
1925 unsigned long count
= 0;
1926 phys_addr_t start
, end
;
1929 memblock_clear_hotplug(0, -1);
1931 for_each_reserved_mem_region(i
, &start
, &end
)
1932 reserve_bootmem_region(start
, end
);
1935 * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
1936 * because in some case like Node0 doesn't have RAM installed
1937 * low ram will be on Node1
1939 for_each_free_mem_range(i
, NUMA_NO_NODE
, MEMBLOCK_NONE
, &start
, &end
,
1941 count
+= __free_memory_core(start
, end
);
1946 static int reset_managed_pages_done __initdata
;
1948 void reset_node_managed_pages(pg_data_t
*pgdat
)
1952 for (z
= pgdat
->node_zones
; z
< pgdat
->node_zones
+ MAX_NR_ZONES
; z
++)
1953 atomic_long_set(&z
->managed_pages
, 0);
1956 void __init
reset_all_zones_managed_pages(void)
1958 struct pglist_data
*pgdat
;
1960 if (reset_managed_pages_done
)
1963 for_each_online_pgdat(pgdat
)
1964 reset_node_managed_pages(pgdat
);
1966 reset_managed_pages_done
= 1;
1970 * memblock_free_all - release free pages to the buddy allocator
1972 * Return: the number of pages actually released.
1974 unsigned long __init
memblock_free_all(void)
1976 unsigned long pages
;
1978 reset_all_zones_managed_pages();
1980 pages
= free_low_memory_core_early();
1981 totalram_pages_add(pages
);
1986 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
1988 static int memblock_debug_show(struct seq_file
*m
, void *private)
1990 struct memblock_type
*type
= m
->private;
1991 struct memblock_region
*reg
;
1995 for (i
= 0; i
< type
->cnt
; i
++) {
1996 reg
= &type
->regions
[i
];
1997 end
= reg
->base
+ reg
->size
- 1;
1999 seq_printf(m
, "%4d: ", i
);
2000 seq_printf(m
, "%pa..%pa\n", ®
->base
, &end
);
2004 DEFINE_SHOW_ATTRIBUTE(memblock_debug
);
2006 static int __init
memblock_init_debugfs(void)
2008 struct dentry
*root
= debugfs_create_dir("memblock", NULL
);
2010 debugfs_create_file("memory", 0444, root
,
2011 &memblock
.memory
, &memblock_debug_fops
);
2012 debugfs_create_file("reserved", 0444, root
,
2013 &memblock
.reserved
, &memblock_debug_fops
);
2014 #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
2015 debugfs_create_file("physmem", 0444, root
,
2016 &memblock
.physmem
, &memblock_debug_fops
);
2021 __initcall(memblock_init_debugfs
);
2023 #endif /* CONFIG_DEBUG_FS */