2 * MTRR (Memory Type Range Register) cleanup
4 * Copyright (C) 2009 Yinghai Lu
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the Free
18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 #include <linux/module.h>
21 #include <linux/init.h>
22 #include <linux/pci.h>
23 #include <linux/smp.h>
24 #include <linux/cpu.h>
25 #include <linux/mutex.h>
26 #include <linux/uaccess.h>
27 #include <linux/kvm_para.h>
28 #include <linux/range.h>
30 #include <asm/processor.h>
37 struct var_mtrr_range_state
{
38 unsigned long base_pfn
;
39 unsigned long size_pfn
;
43 struct var_mtrr_state
{
44 unsigned long range_startk
;
45 unsigned long range_sizek
;
46 unsigned long chunk_sizek
;
47 unsigned long gran_sizek
;
51 /* Should be related to MTRR_VAR_RANGES nums */
54 static struct range __initdata range
[RANGE_NUM
];
55 static int __initdata nr_range
;
57 static struct var_mtrr_range_state __initdata range_state
[RANGE_NUM
];
59 static int __initdata debug_print
;
60 #define Dprintk(x...) do { if (debug_print) printk(KERN_DEBUG x); } while (0)
62 #define BIOS_BUG_MSG KERN_WARNING \
63 "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n"
66 x86_get_mtrr_mem_range(struct range
*range
, int nr_range
,
67 unsigned long extra_remove_base
,
68 unsigned long extra_remove_size
)
70 unsigned long base
, size
;
74 for (i
= 0; i
< num_var_ranges
; i
++) {
75 type
= range_state
[i
].type
;
76 if (type
!= MTRR_TYPE_WRBACK
)
78 base
= range_state
[i
].base_pfn
;
79 size
= range_state
[i
].size_pfn
;
80 nr_range
= add_range_with_merge(range
, RANGE_NUM
, nr_range
,
84 printk(KERN_DEBUG
"After WB checking\n");
85 for (i
= 0; i
< nr_range
; i
++)
86 printk(KERN_DEBUG
"MTRR MAP PFN: %016llx - %016llx\n",
87 range
[i
].start
, range
[i
].end
);
90 /* Take out UC ranges: */
91 for (i
= 0; i
< num_var_ranges
; i
++) {
92 type
= range_state
[i
].type
;
93 if (type
!= MTRR_TYPE_UNCACHABLE
&&
94 type
!= MTRR_TYPE_WRPROT
)
96 size
= range_state
[i
].size_pfn
;
99 base
= range_state
[i
].base_pfn
;
100 if (base
< (1<<(20-PAGE_SHIFT
)) && mtrr_state
.have_fixed
&&
101 (mtrr_state
.enabled
& 1)) {
102 /* Var MTRR contains UC entry below 1M? Skip it: */
103 printk(BIOS_BUG_MSG
, i
);
104 if (base
+ size
<= (1<<(20-PAGE_SHIFT
)))
106 size
-= (1<<(20-PAGE_SHIFT
)) - base
;
107 base
= 1<<(20-PAGE_SHIFT
);
109 subtract_range(range
, RANGE_NUM
, base
, base
+ size
);
111 if (extra_remove_size
)
112 subtract_range(range
, RANGE_NUM
, extra_remove_base
,
113 extra_remove_base
+ extra_remove_size
);
116 printk(KERN_DEBUG
"After UC checking\n");
117 for (i
= 0; i
< RANGE_NUM
; i
++) {
120 printk(KERN_DEBUG
"MTRR MAP PFN: %016llx - %016llx\n",
121 range
[i
].start
, range
[i
].end
);
125 /* sort the ranges */
126 nr_range
= clean_sort_range(range
, RANGE_NUM
);
128 printk(KERN_DEBUG
"After sorting\n");
129 for (i
= 0; i
< nr_range
; i
++)
130 printk(KERN_DEBUG
"MTRR MAP PFN: %016llx - %016llx\n",
131 range
[i
].start
, range
[i
].end
);
137 #ifdef CONFIG_MTRR_SANITIZER
139 static unsigned long __init
sum_ranges(struct range
*range
, int nr_range
)
141 unsigned long sum
= 0;
144 for (i
= 0; i
< nr_range
; i
++)
145 sum
+= range
[i
].end
- range
[i
].start
;
150 static int enable_mtrr_cleanup __initdata
=
151 CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT
;
153 static int __init
disable_mtrr_cleanup_setup(char *str
)
155 enable_mtrr_cleanup
= 0;
158 early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup
);
160 static int __init
enable_mtrr_cleanup_setup(char *str
)
162 enable_mtrr_cleanup
= 1;
165 early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup
);
167 static int __init
mtrr_cleanup_debug_setup(char *str
)
172 early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup
);
175 set_var_mtrr(unsigned int reg
, unsigned long basek
, unsigned long sizek
,
176 unsigned char type
, unsigned int address_bits
)
178 u32 base_lo
, base_hi
, mask_lo
, mask_hi
;
182 fill_mtrr_var_range(reg
, 0, 0, 0, 0);
186 mask
= (1ULL << address_bits
) - 1;
187 mask
&= ~((((u64
)sizek
) << 10) - 1);
189 base
= ((u64
)basek
) << 10;
194 base_lo
= base
& ((1ULL<<32) - 1);
195 base_hi
= base
>> 32;
197 mask_lo
= mask
& ((1ULL<<32) - 1);
198 mask_hi
= mask
>> 32;
200 fill_mtrr_var_range(reg
, base_lo
, base_hi
, mask_lo
, mask_hi
);
204 save_var_mtrr(unsigned int reg
, unsigned long basek
, unsigned long sizek
,
207 range_state
[reg
].base_pfn
= basek
>> (PAGE_SHIFT
- 10);
208 range_state
[reg
].size_pfn
= sizek
>> (PAGE_SHIFT
- 10);
209 range_state
[reg
].type
= type
;
212 static void __init
set_var_mtrr_all(unsigned int address_bits
)
214 unsigned long basek
, sizek
;
218 for (reg
= 0; reg
< num_var_ranges
; reg
++) {
219 basek
= range_state
[reg
].base_pfn
<< (PAGE_SHIFT
- 10);
220 sizek
= range_state
[reg
].size_pfn
<< (PAGE_SHIFT
- 10);
221 type
= range_state
[reg
].type
;
223 set_var_mtrr(reg
, basek
, sizek
, type
, address_bits
);
227 static unsigned long to_size_factor(unsigned long sizek
, char *factorp
)
229 unsigned long base
= sizek
;
232 if (base
& ((1<<10) - 1)) {
233 /* Not MB-aligned: */
235 } else if (base
& ((1<<20) - 1)) {
248 static unsigned int __init
249 range_to_mtrr(unsigned int reg
, unsigned long range_startk
,
250 unsigned long range_sizek
, unsigned char type
)
252 if (!range_sizek
|| (reg
>= num_var_ranges
))
255 while (range_sizek
) {
256 unsigned long max_align
, align
;
259 /* Compute the maximum size with which we can make a range: */
261 max_align
= __ffs(range_startk
);
263 max_align
= BITS_PER_LONG
- 1;
265 align
= __fls(range_sizek
);
266 if (align
> max_align
)
269 sizek
= 1UL << align
;
271 char start_factor
= 'K', size_factor
= 'K';
272 unsigned long start_base
, size_base
;
274 start_base
= to_size_factor(range_startk
, &start_factor
);
275 size_base
= to_size_factor(sizek
, &size_factor
);
277 Dprintk("Setting variable MTRR %d, "
278 "base: %ld%cB, range: %ld%cB, type %s\n",
279 reg
, start_base
, start_factor
,
280 size_base
, size_factor
,
281 (type
== MTRR_TYPE_UNCACHABLE
) ? "UC" :
282 ((type
== MTRR_TYPE_WRBACK
) ? "WB" : "Other")
285 save_var_mtrr(reg
++, range_startk
, sizek
, type
);
286 range_startk
+= sizek
;
287 range_sizek
-= sizek
;
288 if (reg
>= num_var_ranges
)
294 static unsigned __init
295 range_to_mtrr_with_hole(struct var_mtrr_state
*state
, unsigned long basek
,
298 unsigned long hole_basek
, hole_sizek
;
299 unsigned long second_basek
, second_sizek
;
300 unsigned long range0_basek
, range0_sizek
;
301 unsigned long range_basek
, range_sizek
;
302 unsigned long chunk_sizek
;
303 unsigned long gran_sizek
;
309 chunk_sizek
= state
->chunk_sizek
;
310 gran_sizek
= state
->gran_sizek
;
312 /* Align with gran size, prevent small block used up MTRRs: */
313 range_basek
= ALIGN(state
->range_startk
, gran_sizek
);
314 if ((range_basek
> basek
) && basek
)
317 state
->range_sizek
-= (range_basek
- state
->range_startk
);
318 range_sizek
= ALIGN(state
->range_sizek
, gran_sizek
);
320 while (range_sizek
> state
->range_sizek
) {
321 range_sizek
-= gran_sizek
;
325 state
->range_sizek
= range_sizek
;
327 /* Try to append some small hole: */
328 range0_basek
= state
->range_startk
;
329 range0_sizek
= ALIGN(state
->range_sizek
, chunk_sizek
);
332 if (range0_sizek
== state
->range_sizek
) {
333 Dprintk("rangeX: %016lx - %016lx\n",
335 (range0_basek
+ state
->range_sizek
)<<10);
336 state
->reg
= range_to_mtrr(state
->reg
, range0_basek
,
337 state
->range_sizek
, MTRR_TYPE_WRBACK
);
341 /* Only cut back when it is not the last: */
343 while (range0_basek
+ range0_sizek
> (basek
+ sizek
)) {
344 if (range0_sizek
>= chunk_sizek
)
345 range0_sizek
-= chunk_sizek
;
355 range_basek
= range0_basek
+ range0_sizek
;
357 /* One hole in the middle: */
358 if (range_basek
> basek
&& range_basek
<= (basek
+ sizek
))
359 second_sizek
= range_basek
- basek
;
361 if (range0_sizek
> state
->range_sizek
) {
363 /* One hole in middle or at the end: */
364 hole_sizek
= range0_sizek
- state
->range_sizek
- second_sizek
;
366 /* Hole size should be less than half of range0 size: */
367 if (hole_sizek
>= (range0_sizek
>> 1) &&
368 range0_sizek
>= chunk_sizek
) {
369 range0_sizek
-= chunk_sizek
;
378 Dprintk("range0: %016lx - %016lx\n",
380 (range0_basek
+ range0_sizek
)<<10);
381 state
->reg
= range_to_mtrr(state
->reg
, range0_basek
,
382 range0_sizek
, MTRR_TYPE_WRBACK
);
385 if (range0_sizek
< state
->range_sizek
) {
386 /* Need to handle left over range: */
387 range_sizek
= state
->range_sizek
- range0_sizek
;
389 Dprintk("range: %016lx - %016lx\n",
391 (range_basek
+ range_sizek
)<<10);
393 state
->reg
= range_to_mtrr(state
->reg
, range_basek
,
394 range_sizek
, MTRR_TYPE_WRBACK
);
398 hole_basek
= range_basek
- hole_sizek
- second_sizek
;
399 Dprintk("hole: %016lx - %016lx\n",
401 (hole_basek
+ hole_sizek
)<<10);
402 state
->reg
= range_to_mtrr(state
->reg
, hole_basek
,
403 hole_sizek
, MTRR_TYPE_UNCACHABLE
);
410 set_var_mtrr_range(struct var_mtrr_state
*state
, unsigned long base_pfn
,
411 unsigned long size_pfn
)
413 unsigned long basek
, sizek
;
414 unsigned long second_sizek
= 0;
416 if (state
->reg
>= num_var_ranges
)
419 basek
= base_pfn
<< (PAGE_SHIFT
- 10);
420 sizek
= size_pfn
<< (PAGE_SHIFT
- 10);
422 /* See if I can merge with the last range: */
423 if ((basek
<= 1024) ||
424 (state
->range_startk
+ state
->range_sizek
== basek
)) {
425 unsigned long endk
= basek
+ sizek
;
426 state
->range_sizek
= endk
- state
->range_startk
;
429 /* Write the range mtrrs: */
430 if (state
->range_sizek
!= 0)
431 second_sizek
= range_to_mtrr_with_hole(state
, basek
, sizek
);
433 /* Allocate an msr: */
434 state
->range_startk
= basek
+ second_sizek
;
435 state
->range_sizek
= sizek
- second_sizek
;
438 /* Mininum size of mtrr block that can take hole: */
439 static u64 mtrr_chunk_size __initdata
= (256ULL<<20);
441 static int __init
parse_mtrr_chunk_size_opt(char *p
)
445 mtrr_chunk_size
= memparse(p
, &p
);
448 early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt
);
450 /* Granularity of mtrr of block: */
451 static u64 mtrr_gran_size __initdata
;
453 static int __init
parse_mtrr_gran_size_opt(char *p
)
457 mtrr_gran_size
= memparse(p
, &p
);
460 early_param("mtrr_gran_size", parse_mtrr_gran_size_opt
);
462 static unsigned long nr_mtrr_spare_reg __initdata
=
463 CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT
;
465 static int __init
parse_mtrr_spare_reg(char *arg
)
468 nr_mtrr_spare_reg
= simple_strtoul(arg
, NULL
, 0);
471 early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg
);
474 x86_setup_var_mtrrs(struct range
*range
, int nr_range
,
475 u64 chunk_size
, u64 gran_size
)
477 struct var_mtrr_state var_state
;
481 var_state
.range_startk
= 0;
482 var_state
.range_sizek
= 0;
484 var_state
.chunk_sizek
= chunk_size
>> 10;
485 var_state
.gran_sizek
= gran_size
>> 10;
487 memset(range_state
, 0, sizeof(range_state
));
489 /* Write the range: */
490 for (i
= 0; i
< nr_range
; i
++) {
491 set_var_mtrr_range(&var_state
, range
[i
].start
,
492 range
[i
].end
- range
[i
].start
);
495 /* Write the last range: */
496 if (var_state
.range_sizek
!= 0)
497 range_to_mtrr_with_hole(&var_state
, 0, 0);
499 num_reg
= var_state
.reg
;
500 /* Clear out the extra MTRR's: */
501 while (var_state
.reg
< num_var_ranges
) {
502 save_var_mtrr(var_state
.reg
, 0, 0, 0);
509 struct mtrr_cleanup_result
{
510 unsigned long gran_sizek
;
511 unsigned long chunk_sizek
;
512 unsigned long lose_cover_sizek
;
513 unsigned int num_reg
;
518 * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G
519 * chunk size: gran_size, ..., 2G
520 * so we need (1+16)*8
522 #define NUM_RESULT 136
523 #define PSHIFT (PAGE_SHIFT - 10)
525 static struct mtrr_cleanup_result __initdata result
[NUM_RESULT
];
526 static unsigned long __initdata min_loss_pfn
[RANGE_NUM
];
528 static void __init
print_out_mtrr_range_state(void)
530 char start_factor
= 'K', size_factor
= 'K';
531 unsigned long start_base
, size_base
;
535 for (i
= 0; i
< num_var_ranges
; i
++) {
537 size_base
= range_state
[i
].size_pfn
<< (PAGE_SHIFT
- 10);
541 size_base
= to_size_factor(size_base
, &size_factor
),
542 start_base
= range_state
[i
].base_pfn
<< (PAGE_SHIFT
- 10);
543 start_base
= to_size_factor(start_base
, &start_factor
),
544 type
= range_state
[i
].type
;
546 printk(KERN_DEBUG
"reg %d, base: %ld%cB, range: %ld%cB, type %s\n",
547 i
, start_base
, start_factor
,
548 size_base
, size_factor
,
549 (type
== MTRR_TYPE_UNCACHABLE
) ? "UC" :
550 ((type
== MTRR_TYPE_WRPROT
) ? "WP" :
551 ((type
== MTRR_TYPE_WRBACK
) ? "WB" : "Other"))
556 static int __init
mtrr_need_cleanup(void)
561 /* Extra one for all 0: */
562 int num
[MTRR_NUM_TYPES
+ 1];
564 /* Check entries number: */
565 memset(num
, 0, sizeof(num
));
566 for (i
= 0; i
< num_var_ranges
; i
++) {
567 type
= range_state
[i
].type
;
568 size
= range_state
[i
].size_pfn
;
569 if (type
>= MTRR_NUM_TYPES
)
572 type
= MTRR_NUM_TYPES
;
576 /* Check if we got UC entries: */
577 if (!num
[MTRR_TYPE_UNCACHABLE
])
580 /* Check if we only had WB and UC */
581 if (num
[MTRR_TYPE_WRBACK
] + num
[MTRR_TYPE_UNCACHABLE
] !=
582 num_var_ranges
- num
[MTRR_NUM_TYPES
])
588 static unsigned long __initdata range_sums
;
591 mtrr_calc_range_state(u64 chunk_size
, u64 gran_size
,
592 unsigned long x_remove_base
,
593 unsigned long x_remove_size
, int i
)
595 static struct range range_new
[RANGE_NUM
];
596 unsigned long range_sums_new
;
597 static int nr_range_new
;
600 /* Convert ranges to var ranges state: */
601 num_reg
= x86_setup_var_mtrrs(range
, nr_range
, chunk_size
, gran_size
);
603 /* We got new setting in range_state, check it: */
604 memset(range_new
, 0, sizeof(range_new
));
605 nr_range_new
= x86_get_mtrr_mem_range(range_new
, 0,
606 x_remove_base
, x_remove_size
);
607 range_sums_new
= sum_ranges(range_new
, nr_range_new
);
609 result
[i
].chunk_sizek
= chunk_size
>> 10;
610 result
[i
].gran_sizek
= gran_size
>> 10;
611 result
[i
].num_reg
= num_reg
;
613 if (range_sums
< range_sums_new
) {
614 result
[i
].lose_cover_sizek
= (range_sums_new
- range_sums
) << PSHIFT
;
617 result
[i
].lose_cover_sizek
= (range_sums
- range_sums_new
) << PSHIFT
;
620 /* Double check it: */
621 if (!result
[i
].bad
&& !result
[i
].lose_cover_sizek
) {
622 if (nr_range_new
!= nr_range
|| memcmp(range
, range_new
, sizeof(range
)))
626 if (!result
[i
].bad
&& (range_sums
- range_sums_new
< min_loss_pfn
[num_reg
]))
627 min_loss_pfn
[num_reg
] = range_sums
- range_sums_new
;
630 static void __init
mtrr_print_out_one_result(int i
)
632 unsigned long gran_base
, chunk_base
, lose_base
;
633 char gran_factor
, chunk_factor
, lose_factor
;
635 gran_base
= to_size_factor(result
[i
].gran_sizek
, &gran_factor
);
636 chunk_base
= to_size_factor(result
[i
].chunk_sizek
, &chunk_factor
);
637 lose_base
= to_size_factor(result
[i
].lose_cover_sizek
, &lose_factor
);
639 pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t",
640 result
[i
].bad
? "*BAD*" : " ",
641 gran_base
, gran_factor
, chunk_base
, chunk_factor
);
642 pr_cont("num_reg: %d \tlose cover RAM: %s%ld%c\n",
643 result
[i
].num_reg
, result
[i
].bad
? "-" : "",
644 lose_base
, lose_factor
);
647 static int __init
mtrr_search_optimal_index(void)
653 if (nr_mtrr_spare_reg
>= num_var_ranges
)
654 nr_mtrr_spare_reg
= num_var_ranges
- 1;
657 for (i
= num_var_ranges
- nr_mtrr_spare_reg
; i
> 0; i
--) {
658 if (!min_loss_pfn
[i
])
663 if (num_reg_good
!= -1) {
664 for (i
= 0; i
< NUM_RESULT
; i
++) {
665 if (!result
[i
].bad
&&
666 result
[i
].num_reg
== num_reg_good
&&
667 !result
[i
].lose_cover_sizek
) {
677 int __init
mtrr_cleanup(unsigned address_bits
)
679 unsigned long x_remove_base
, x_remove_size
;
680 unsigned long base
, size
, def
, dummy
;
681 u64 chunk_size
, gran_size
;
686 if (!is_cpu(INTEL
) || enable_mtrr_cleanup
< 1)
689 rdmsr(MSR_MTRRdefType
, def
, dummy
);
691 if (def
!= MTRR_TYPE_UNCACHABLE
)
694 /* Get it and store it aside: */
695 memset(range_state
, 0, sizeof(range_state
));
696 for (i
= 0; i
< num_var_ranges
; i
++) {
697 mtrr_if
->get(i
, &base
, &size
, &type
);
698 range_state
[i
].base_pfn
= base
;
699 range_state
[i
].size_pfn
= size
;
700 range_state
[i
].type
= type
;
703 /* Check if we need handle it and can handle it: */
704 if (!mtrr_need_cleanup())
707 /* Print original var MTRRs at first, for debugging: */
708 printk(KERN_DEBUG
"original variable MTRRs\n");
709 print_out_mtrr_range_state();
711 memset(range
, 0, sizeof(range
));
713 x_remove_base
= 1 << (32 - PAGE_SHIFT
);
715 x_remove_size
= (mtrr_tom2
>> PAGE_SHIFT
) - x_remove_base
;
717 nr_range
= x86_get_mtrr_mem_range(range
, 0, x_remove_base
, x_remove_size
);
719 * [0, 1M) should always be covered by var mtrr with WB
720 * and fixed mtrrs should take effect before var mtrr for it:
722 nr_range
= add_range_with_merge(range
, RANGE_NUM
, nr_range
, 0,
723 1ULL<<(20 - PAGE_SHIFT
));
724 /* Sort the ranges: */
725 sort_range(range
, nr_range
);
727 range_sums
= sum_ranges(range
, nr_range
);
728 printk(KERN_INFO
"total RAM covered: %ldM\n",
729 range_sums
>> (20 - PAGE_SHIFT
));
731 if (mtrr_chunk_size
&& mtrr_gran_size
) {
733 mtrr_calc_range_state(mtrr_chunk_size
, mtrr_gran_size
,
734 x_remove_base
, x_remove_size
, i
);
736 mtrr_print_out_one_result(i
);
738 if (!result
[i
].bad
) {
739 set_var_mtrr_all(address_bits
);
740 printk(KERN_DEBUG
"New variable MTRRs\n");
741 print_out_mtrr_range_state();
744 printk(KERN_INFO
"invalid mtrr_gran_size or mtrr_chunk_size, "
745 "will find optimal one\n");
749 memset(min_loss_pfn
, 0xff, sizeof(min_loss_pfn
));
750 memset(result
, 0, sizeof(result
));
751 for (gran_size
= (1ULL<<16); gran_size
< (1ULL<<32); gran_size
<<= 1) {
753 for (chunk_size
= gran_size
; chunk_size
< (1ULL<<32);
759 mtrr_calc_range_state(chunk_size
, gran_size
,
760 x_remove_base
, x_remove_size
, i
);
762 mtrr_print_out_one_result(i
);
763 printk(KERN_INFO
"\n");
770 /* Try to find the optimal index: */
771 index_good
= mtrr_search_optimal_index();
773 if (index_good
!= -1) {
774 printk(KERN_INFO
"Found optimal setting for mtrr clean up\n");
776 mtrr_print_out_one_result(i
);
778 /* Convert ranges to var ranges state: */
779 chunk_size
= result
[i
].chunk_sizek
;
781 gran_size
= result
[i
].gran_sizek
;
783 x86_setup_var_mtrrs(range
, nr_range
, chunk_size
, gran_size
);
784 set_var_mtrr_all(address_bits
);
785 printk(KERN_DEBUG
"New variable MTRRs\n");
786 print_out_mtrr_range_state();
790 for (i
= 0; i
< NUM_RESULT
; i
++)
791 mtrr_print_out_one_result(i
);
794 printk(KERN_INFO
"mtrr_cleanup: can not find optimal value\n");
795 printk(KERN_INFO
"please specify mtrr_gran_size/mtrr_chunk_size\n");
800 int __init
mtrr_cleanup(unsigned address_bits
)
806 static int disable_mtrr_trim
;
808 static int __init
disable_mtrr_trim_setup(char *str
)
810 disable_mtrr_trim
= 1;
813 early_param("disable_mtrr_trim", disable_mtrr_trim_setup
);
816 * Newer AMD K8s and later CPUs have a special magic MSR way to force WB
817 * for memory >4GB. Check for that here.
818 * Note this won't check if the MTRRs < 4GB where the magic bit doesn't
819 * apply to are wrong, but so far we don't know of any such case in the wild.
821 #define Tom2Enabled (1U << 21)
822 #define Tom2ForceMemTypeWB (1U << 22)
824 int __init
amd_special_default_mtrr(void)
828 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_AMD
)
830 if (boot_cpu_data
.x86
< 0xf)
832 /* In case some hypervisor doesn't pass SYSCFG through: */
833 if (rdmsr_safe(MSR_K8_SYSCFG
, &l
, &h
) < 0)
836 * Memory between 4GB and top of mem is forced WB by this magic bit.
837 * Reserved before K8RevF, but should be zero there.
839 if ((l
& (Tom2Enabled
| Tom2ForceMemTypeWB
)) ==
840 (Tom2Enabled
| Tom2ForceMemTypeWB
))
846 real_trim_memory(unsigned long start_pfn
, unsigned long limit_pfn
)
848 u64 trim_start
, trim_size
;
850 trim_start
= start_pfn
;
851 trim_start
<<= PAGE_SHIFT
;
853 trim_size
= limit_pfn
;
854 trim_size
<<= PAGE_SHIFT
;
855 trim_size
-= trim_start
;
857 return e820_update_range(trim_start
, trim_size
, E820_RAM
, E820_RESERVED
);
861 * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs
862 * @end_pfn: ending page frame number
864 * Some buggy BIOSes don't setup the MTRRs properly for systems with certain
865 * memory configurations. This routine checks that the highest MTRR matches
866 * the end of memory, to make sure the MTRRs having a write back type cover
867 * all of the memory the kernel is intending to use. If not, it'll trim any
868 * memory off the end by adjusting end_pfn, removing it from the kernel's
869 * allocation pools, warning the user with an obnoxious message.
871 int __init
mtrr_trim_uncached_memory(unsigned long end_pfn
)
873 unsigned long i
, base
, size
, highest_pfn
= 0, def
, dummy
;
876 /* extra one for all 0 */
877 int num
[MTRR_NUM_TYPES
+ 1];
880 * Make sure we only trim uncachable memory on machines that
881 * support the Intel MTRR architecture:
883 if (!is_cpu(INTEL
) || disable_mtrr_trim
)
886 rdmsr(MSR_MTRRdefType
, def
, dummy
);
888 if (def
!= MTRR_TYPE_UNCACHABLE
)
891 /* Get it and store it aside: */
892 memset(range_state
, 0, sizeof(range_state
));
893 for (i
= 0; i
< num_var_ranges
; i
++) {
894 mtrr_if
->get(i
, &base
, &size
, &type
);
895 range_state
[i
].base_pfn
= base
;
896 range_state
[i
].size_pfn
= size
;
897 range_state
[i
].type
= type
;
900 /* Find highest cached pfn: */
901 for (i
= 0; i
< num_var_ranges
; i
++) {
902 type
= range_state
[i
].type
;
903 if (type
!= MTRR_TYPE_WRBACK
)
905 base
= range_state
[i
].base_pfn
;
906 size
= range_state
[i
].size_pfn
;
907 if (highest_pfn
< base
+ size
)
908 highest_pfn
= base
+ size
;
911 /* kvm/qemu doesn't have mtrr set right, don't trim them all: */
913 printk(KERN_INFO
"CPU MTRRs all blank - virtualized system.\n");
917 /* Check entries number: */
918 memset(num
, 0, sizeof(num
));
919 for (i
= 0; i
< num_var_ranges
; i
++) {
920 type
= range_state
[i
].type
;
921 if (type
>= MTRR_NUM_TYPES
)
923 size
= range_state
[i
].size_pfn
;
925 type
= MTRR_NUM_TYPES
;
929 /* No entry for WB? */
930 if (!num
[MTRR_TYPE_WRBACK
])
933 /* Check if we only had WB and UC: */
934 if (num
[MTRR_TYPE_WRBACK
] + num
[MTRR_TYPE_UNCACHABLE
] !=
935 num_var_ranges
- num
[MTRR_NUM_TYPES
])
938 memset(range
, 0, sizeof(range
));
941 range
[nr_range
].start
= (1ULL<<(32 - PAGE_SHIFT
));
942 range
[nr_range
].end
= mtrr_tom2
>> PAGE_SHIFT
;
943 if (highest_pfn
< range
[nr_range
].end
)
944 highest_pfn
= range
[nr_range
].end
;
947 nr_range
= x86_get_mtrr_mem_range(range
, nr_range
, 0, 0);
949 /* Check the head: */
952 total_trim_size
+= real_trim_memory(0, range
[0].start
);
954 /* Check the holes: */
955 for (i
= 0; i
< nr_range
- 1; i
++) {
956 if (range
[i
].end
< range
[i
+1].start
)
957 total_trim_size
+= real_trim_memory(range
[i
].end
,
963 if (range
[i
].end
< end_pfn
)
964 total_trim_size
+= real_trim_memory(range
[i
].end
,
967 if (total_trim_size
) {
968 pr_warning("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", total_trim_size
>> 20);
970 if (!changed_by_mtrr_cleanup
)
973 pr_info("update e820 for mtrr\n");