1 // SPDX-License-Identifier: GPL-2.0-only
3 * AMD Memory Encryption Support
5 * Copyright (C) 2016-2024 Advanced Micro Devices, Inc.
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
10 #define DISABLE_BRANCH_PROFILING
12 #include <linux/linkage.h>
13 #include <linux/init.h>
15 #include <linux/dma-direct.h>
16 #include <linux/swiotlb.h>
17 #include <linux/mem_encrypt.h>
18 #include <linux/device.h>
19 #include <linux/kernel.h>
20 #include <linux/bitops.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/cc_platform.h>
24 #include <asm/tlbflush.h>
25 #include <asm/fixmap.h>
26 #include <asm/setup.h>
27 #include <asm/mem_encrypt.h>
28 #include <asm/bootparam.h>
29 #include <asm/set_memory.h>
30 #include <asm/cacheflush.h>
31 #include <asm/processor-flags.h>
33 #include <asm/cmdline.h>
37 #include "mm_internal.h"
40 * Since SME related variables are set early in the boot process they must
41 * reside in the .data section so as not to be zeroed out when the .bss
42 * section is later cleared.
44 u64 sme_me_mask
__section(".data") = 0;
45 u64 sev_status
__section(".data") = 0;
46 u64 sev_check_data
__section(".data") = 0;
47 EXPORT_SYMBOL(sme_me_mask
);
49 /* Buffer used for early in-place encryption by BSP, no locking needed */
50 static char sme_early_buffer
[PAGE_SIZE
] __initdata
__aligned(PAGE_SIZE
);
53 * SNP-specific routine which needs to additionally change the page state from
54 * private to shared before copying the data from the source to destination and
55 * restore after the copy.
57 static inline void __init
snp_memcpy(void *dst
, void *src
, size_t sz
,
58 unsigned long paddr
, bool decrypt
)
60 unsigned long npages
= PAGE_ALIGN(sz
) >> PAGE_SHIFT
;
64 * @paddr needs to be accessed decrypted, mark the page shared in
65 * the RMP table before copying it.
67 early_snp_set_memory_shared((unsigned long)__va(paddr
), paddr
, npages
);
71 /* Restore the page state after the memcpy. */
72 early_snp_set_memory_private((unsigned long)__va(paddr
), paddr
, npages
);
75 * @paddr need to be accessed encrypted, no need for the page state
83 * This routine does not change the underlying encryption setting of the
84 * page(s) that map this memory. It assumes that eventually the memory is
85 * meant to be accessed as either encrypted or decrypted but the contents
86 * are currently not in the desired state.
88 * This routine follows the steps outlined in the AMD64 Architecture
89 * Programmer's Manual Volume 2, Section 7.10.8 Encrypt-in-Place.
91 static void __init
__sme_early_enc_dec(resource_size_t paddr
,
92 unsigned long size
, bool enc
)
103 * There are limited number of early mapping slots, so map (at most)
107 len
= min_t(size_t, sizeof(sme_early_buffer
), size
);
110 * Create mappings for the current and desired format of
111 * the memory. Use a write-protected mapping for the source.
113 src
= enc
? early_memremap_decrypted_wp(paddr
, len
) :
114 early_memremap_encrypted_wp(paddr
, len
);
116 dst
= enc
? early_memremap_encrypted(paddr
, len
) :
117 early_memremap_decrypted(paddr
, len
);
120 * If a mapping can't be obtained to perform the operation,
121 * then eventual access of that area in the desired mode
122 * will cause a crash.
124 BUG_ON(!src
|| !dst
);
127 * Use a temporary buffer, of cache-line multiple size, to
128 * avoid data corruption as documented in the APM.
130 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP
)) {
131 snp_memcpy(sme_early_buffer
, src
, len
, paddr
, enc
);
132 snp_memcpy(dst
, sme_early_buffer
, len
, paddr
, !enc
);
134 memcpy(sme_early_buffer
, src
, len
);
135 memcpy(dst
, sme_early_buffer
, len
);
138 early_memunmap(dst
, len
);
139 early_memunmap(src
, len
);
146 void __init
sme_early_encrypt(resource_size_t paddr
, unsigned long size
)
148 __sme_early_enc_dec(paddr
, size
, true);
151 void __init
sme_early_decrypt(resource_size_t paddr
, unsigned long size
)
153 __sme_early_enc_dec(paddr
, size
, false);
156 static void __init
__sme_early_map_unmap_mem(void *vaddr
, unsigned long size
,
159 unsigned long paddr
= (unsigned long)vaddr
- __PAGE_OFFSET
;
160 pmdval_t pmd_flags
, pmd
;
162 /* Use early_pmd_flags but remove the encryption mask */
163 pmd_flags
= __sme_clr(early_pmd_flags
);
166 pmd
= map
? (paddr
& PMD_MASK
) + pmd_flags
: 0;
167 __early_make_pgtable((unsigned long)vaddr
, pmd
);
171 size
= (size
<= PMD_SIZE
) ? 0 : size
- PMD_SIZE
;
177 void __init
sme_unmap_bootdata(char *real_mode_data
)
179 struct boot_params
*boot_data
;
180 unsigned long cmdline_paddr
;
182 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT
))
185 /* Get the command line address before unmapping the real_mode_data */
186 boot_data
= (struct boot_params
*)real_mode_data
;
187 cmdline_paddr
= boot_data
->hdr
.cmd_line_ptr
| ((u64
)boot_data
->ext_cmd_line_ptr
<< 32);
189 __sme_early_map_unmap_mem(real_mode_data
, sizeof(boot_params
), false);
194 __sme_early_map_unmap_mem(__va(cmdline_paddr
), COMMAND_LINE_SIZE
, false);
197 void __init
sme_map_bootdata(char *real_mode_data
)
199 struct boot_params
*boot_data
;
200 unsigned long cmdline_paddr
;
202 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT
))
205 __sme_early_map_unmap_mem(real_mode_data
, sizeof(boot_params
), true);
207 /* Get the command line address after mapping the real_mode_data */
208 boot_data
= (struct boot_params
*)real_mode_data
;
209 cmdline_paddr
= boot_data
->hdr
.cmd_line_ptr
| ((u64
)boot_data
->ext_cmd_line_ptr
<< 32);
214 __sme_early_map_unmap_mem(__va(cmdline_paddr
), COMMAND_LINE_SIZE
, true);
217 static unsigned long pg_level_to_pfn(int level
, pte_t
*kpte
, pgprot_t
*ret_prot
)
219 unsigned long pfn
= 0;
224 pfn
= pte_pfn(*kpte
);
225 prot
= pte_pgprot(*kpte
);
228 pfn
= pmd_pfn(*(pmd_t
*)kpte
);
229 prot
= pmd_pgprot(*(pmd_t
*)kpte
);
232 pfn
= pud_pfn(*(pud_t
*)kpte
);
233 prot
= pud_pgprot(*(pud_t
*)kpte
);
236 WARN_ONCE(1, "Invalid level for kpte\n");
246 static bool amd_enc_tlb_flush_required(bool enc
)
251 static bool amd_enc_cache_flush_required(void)
253 return !cpu_feature_enabled(X86_FEATURE_SME_COHERENT
);
256 static void enc_dec_hypercall(unsigned long vaddr
, unsigned long size
, bool enc
)
258 #ifdef CONFIG_PARAVIRT
259 unsigned long vaddr_end
= vaddr
+ size
;
261 while (vaddr
< vaddr_end
) {
262 int psize
, pmask
, level
;
266 kpte
= lookup_address(vaddr
, &level
);
267 if (!kpte
|| pte_none(*kpte
)) {
268 WARN_ONCE(1, "kpte lookup for vaddr\n");
272 pfn
= pg_level_to_pfn(level
, kpte
, NULL
);
276 psize
= page_level_size(level
);
277 pmask
= page_level_mask(level
);
279 notify_page_enc_status_changed(pfn
, psize
>> PAGE_SHIFT
, enc
);
281 vaddr
= (vaddr
& pmask
) + psize
;
286 static int amd_enc_status_change_prepare(unsigned long vaddr
, int npages
, bool enc
)
289 * To maintain the security guarantees of SEV-SNP guests, make sure
290 * to invalidate the memory before encryption attribute is cleared.
292 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP
) && !enc
)
293 snp_set_memory_shared(vaddr
, npages
);
298 /* Return true unconditionally: return value doesn't matter for the SEV side */
299 static int amd_enc_status_change_finish(unsigned long vaddr
, int npages
, bool enc
)
302 * After memory is mapped encrypted in the page table, validate it
303 * so that it is consistent with the page table updates.
305 if (cc_platform_has(CC_ATTR_GUEST_SEV_SNP
) && enc
)
306 snp_set_memory_private(vaddr
, npages
);
308 if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT
))
309 enc_dec_hypercall(vaddr
, npages
<< PAGE_SHIFT
, enc
);
314 static void __init
__set_clr_pte_enc(pte_t
*kpte
, int level
, bool enc
)
316 pgprot_t old_prot
, new_prot
;
317 unsigned long pfn
, pa
, size
;
320 pfn
= pg_level_to_pfn(level
, kpte
, &old_prot
);
326 pgprot_val(new_prot
) |= _PAGE_ENC
;
328 pgprot_val(new_prot
) &= ~_PAGE_ENC
;
330 /* If prot is same then do nothing. */
331 if (pgprot_val(old_prot
) == pgprot_val(new_prot
))
334 pa
= pfn
<< PAGE_SHIFT
;
335 size
= page_level_size(level
);
338 * We are going to perform in-place en-/decryption and change the
339 * physical page attribute from C=1 to C=0 or vice versa. Flush the
340 * caches to ensure that data gets accessed with the correct C-bit.
342 clflush_cache_range(__va(pa
), size
);
344 /* Encrypt/decrypt the contents in-place */
346 sme_early_encrypt(pa
, size
);
348 sme_early_decrypt(pa
, size
);
351 * ON SNP, the page state in the RMP table must happen
352 * before the page table updates.
354 early_snp_set_memory_shared((unsigned long)__va(pa
), pa
, 1);
357 /* Change the page encryption mask. */
358 new_pte
= pfn_pte(pfn
, new_prot
);
359 set_pte_atomic(kpte
, new_pte
);
362 * If page is set encrypted in the page table, then update the RMP table to
363 * add this page as private.
366 early_snp_set_memory_private((unsigned long)__va(pa
), pa
, 1);
369 static int __init
early_set_memory_enc_dec(unsigned long vaddr
,
370 unsigned long size
, bool enc
)
372 unsigned long vaddr_end
, vaddr_next
, start
;
373 unsigned long psize
, pmask
;
374 int split_page_size_mask
;
380 vaddr_end
= vaddr
+ size
;
382 for (; vaddr
< vaddr_end
; vaddr
= vaddr_next
) {
383 kpte
= lookup_address(vaddr
, &level
);
384 if (!kpte
|| pte_none(*kpte
)) {
389 if (level
== PG_LEVEL_4K
) {
390 __set_clr_pte_enc(kpte
, level
, enc
);
391 vaddr_next
= (vaddr
& PAGE_MASK
) + PAGE_SIZE
;
395 psize
= page_level_size(level
);
396 pmask
= page_level_mask(level
);
399 * Check whether we can change the large page in one go.
400 * We request a split when the address is not aligned and
401 * the number of pages to set/clear encryption bit is smaller
402 * than the number of pages in the large page.
404 if (vaddr
== (vaddr
& pmask
) &&
405 ((vaddr_end
- vaddr
) >= psize
)) {
406 __set_clr_pte_enc(kpte
, level
, enc
);
407 vaddr_next
= (vaddr
& pmask
) + psize
;
412 * The virtual address is part of a larger page, create the next
413 * level page table mapping (4K or 2M). If it is part of a 2M
414 * page then we request a split of the large page into 4K
415 * chunks. A 1GB large page is split into 2M pages, resp.
417 if (level
== PG_LEVEL_2M
)
418 split_page_size_mask
= 0;
420 split_page_size_mask
= 1 << PG_LEVEL_2M
;
423 * kernel_physical_mapping_change() does not flush the TLBs, so
424 * a TLB flush is required after we exit from the for loop.
426 kernel_physical_mapping_change(__pa(vaddr
& pmask
),
427 __pa((vaddr_end
& pmask
) + psize
),
428 split_page_size_mask
);
433 early_set_mem_enc_dec_hypercall(start
, size
, enc
);
439 int __init
early_set_memory_decrypted(unsigned long vaddr
, unsigned long size
)
441 return early_set_memory_enc_dec(vaddr
, size
, false);
444 int __init
early_set_memory_encrypted(unsigned long vaddr
, unsigned long size
)
446 return early_set_memory_enc_dec(vaddr
, size
, true);
449 void __init
early_set_mem_enc_dec_hypercall(unsigned long vaddr
, unsigned long size
, bool enc
)
451 enc_dec_hypercall(vaddr
, size
, enc
);
454 void __init
sme_early_init(void)
459 early_pmd_flags
= __sme_set(early_pmd_flags
);
461 __supported_pte_mask
= __sme_set(__supported_pte_mask
);
463 /* Update the protection map with memory encryption mask */
464 add_encrypt_protection_map();
466 x86_platform
.guest
.enc_status_change_prepare
= amd_enc_status_change_prepare
;
467 x86_platform
.guest
.enc_status_change_finish
= amd_enc_status_change_finish
;
468 x86_platform
.guest
.enc_tlb_flush_required
= amd_enc_tlb_flush_required
;
469 x86_platform
.guest
.enc_cache_flush_required
= amd_enc_cache_flush_required
;
472 * AMD-SEV-ES intercepts the RDMSR to read the X2APIC ID in the
473 * parallel bringup low level code. That raises #VC which cannot be
475 * It does not provide a RDMSR GHCB protocol so the early startup
476 * code cannot directly communicate with the secure firmware. The
477 * alternative solution to retrieve the APIC ID via CPUID(0xb),
478 * which is covered by the GHCB protocol, is not viable either
479 * because there is no enforcement of the CPUID(0xb) provided
480 * "initial" APIC ID to be the same as the real APIC ID.
481 * Disable parallel bootup.
483 if (sev_status
& MSR_AMD64_SEV_ES_ENABLED
)
484 x86_cpuinit
.parallel_bringup
= false;
487 * The VMM is capable of injecting interrupt 0x80 and triggering the
488 * compatibility syscall path.
490 * By default, the 32-bit emulation is disabled in order to ensure
491 * the safety of the VM.
493 if (sev_status
& MSR_AMD64_SEV_ENABLED
)
497 * Override init functions that scan the ROM region in SEV-SNP guests,
498 * as this memory is not pre-validated and would thus cause a crash.
500 if (sev_status
& MSR_AMD64_SEV_SNP_ENABLED
) {
501 x86_init
.mpparse
.find_mptable
= x86_init_noop
;
502 x86_init
.pci
.init_irq
= x86_init_noop
;
503 x86_init
.resources
.probe_roms
= x86_init_noop
;
506 * DMI setup behavior for SEV-SNP guests depends on
507 * efi_enabled(EFI_CONFIG_TABLES), which hasn't been
508 * parsed yet. snp_dmi_setup() will run after that
509 * parsing has happened.
511 x86_init
.resources
.dmi_setup
= snp_dmi_setup
;
515 * Switch the SVSM CA mapping (if active) from identity mapped to
518 snp_update_svsm_ca();
521 void __init
mem_encrypt_free_decrypted_mem(void)
523 unsigned long vaddr
, vaddr_end
, npages
;
526 vaddr
= (unsigned long)__start_bss_decrypted_unused
;
527 vaddr_end
= (unsigned long)__end_bss_decrypted
;
528 npages
= (vaddr_end
- vaddr
) >> PAGE_SHIFT
;
531 * If the unused memory range was mapped decrypted, change the encryption
532 * attribute from decrypted to encrypted before freeing it. Base the
533 * re-encryption on the same condition used for the decryption in
534 * sme_postprocess_startup(). Higher level abstractions, such as
535 * CC_ATTR_MEM_ENCRYPT, aren't necessarily equivalent in a Hyper-V VM
536 * using vTOM, where sme_me_mask is always zero.
539 r
= set_memory_encrypted(vaddr
, npages
);
541 pr_warn("failed to free unused decrypted pages\n");
546 free_init_pages("unused decrypted", vaddr
, vaddr_end
);