1 // SPDX-License-Identifier: GPL-2.0
3 * KVM guest address space mapping code
5 * Copyright IBM Corp. 2007, 2016, 2018
6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
7 * David Hildenbrand <david@redhat.com>
8 * Janosch Frank <frankja@linux.vnet.ibm.com>
11 #include <linux/kernel.h>
12 #include <linux/pagewalk.h>
13 #include <linux/swap.h>
14 #include <linux/smp.h>
15 #include <linux/spinlock.h>
16 #include <linux/slab.h>
17 #include <linux/swapops.h>
18 #include <linux/ksm.h>
19 #include <linux/mman.h>
21 #include <asm/pgtable.h>
22 #include <asm/pgalloc.h>
26 #define GMAP_SHADOW_FAKE_TABLE 1ULL
29 * gmap_alloc - allocate and initialize a guest address space
30 * @mm: pointer to the parent mm_struct
31 * @limit: maximum address of the gmap address space
33 * Returns a guest address space structure.
35 static struct gmap
*gmap_alloc(unsigned long limit
)
40 unsigned long etype
, atype
;
42 if (limit
< _REGION3_SIZE
) {
43 limit
= _REGION3_SIZE
- 1;
44 atype
= _ASCE_TYPE_SEGMENT
;
45 etype
= _SEGMENT_ENTRY_EMPTY
;
46 } else if (limit
< _REGION2_SIZE
) {
47 limit
= _REGION2_SIZE
- 1;
48 atype
= _ASCE_TYPE_REGION3
;
49 etype
= _REGION3_ENTRY_EMPTY
;
50 } else if (limit
< _REGION1_SIZE
) {
51 limit
= _REGION1_SIZE
- 1;
52 atype
= _ASCE_TYPE_REGION2
;
53 etype
= _REGION2_ENTRY_EMPTY
;
56 atype
= _ASCE_TYPE_REGION1
;
57 etype
= _REGION1_ENTRY_EMPTY
;
59 gmap
= kzalloc(sizeof(struct gmap
), GFP_KERNEL
);
62 INIT_LIST_HEAD(&gmap
->crst_list
);
63 INIT_LIST_HEAD(&gmap
->children
);
64 INIT_LIST_HEAD(&gmap
->pt_list
);
65 INIT_RADIX_TREE(&gmap
->guest_to_host
, GFP_KERNEL
);
66 INIT_RADIX_TREE(&gmap
->host_to_guest
, GFP_ATOMIC
);
67 INIT_RADIX_TREE(&gmap
->host_to_rmap
, GFP_ATOMIC
);
68 spin_lock_init(&gmap
->guest_table_lock
);
69 spin_lock_init(&gmap
->shadow_lock
);
70 refcount_set(&gmap
->ref_count
, 1);
71 page
= alloc_pages(GFP_KERNEL
, CRST_ALLOC_ORDER
);
75 list_add(&page
->lru
, &gmap
->crst_list
);
76 table
= (unsigned long *) page_to_phys(page
);
77 crst_table_init(table
, etype
);
79 gmap
->asce
= atype
| _ASCE_TABLE_LENGTH
|
80 _ASCE_USER_BITS
| __pa(table
);
81 gmap
->asce_end
= limit
;
91 * gmap_create - create a guest address space
92 * @mm: pointer to the parent mm_struct
93 * @limit: maximum size of the gmap address space
95 * Returns a guest address space structure.
97 struct gmap
*gmap_create(struct mm_struct
*mm
, unsigned long limit
)
100 unsigned long gmap_asce
;
102 gmap
= gmap_alloc(limit
);
106 spin_lock(&mm
->context
.lock
);
107 list_add_rcu(&gmap
->list
, &mm
->context
.gmap_list
);
108 if (list_is_singular(&mm
->context
.gmap_list
))
109 gmap_asce
= gmap
->asce
;
112 WRITE_ONCE(mm
->context
.gmap_asce
, gmap_asce
);
113 spin_unlock(&mm
->context
.lock
);
116 EXPORT_SYMBOL_GPL(gmap_create
);
118 static void gmap_flush_tlb(struct gmap
*gmap
)
120 if (MACHINE_HAS_IDTE
)
121 __tlb_flush_idte(gmap
->asce
);
123 __tlb_flush_global();
126 static void gmap_radix_tree_free(struct radix_tree_root
*root
)
128 struct radix_tree_iter iter
;
129 unsigned long indices
[16];
134 /* A radix tree is freed by deleting all of its entries */
138 radix_tree_for_each_slot(slot
, root
, &iter
, index
) {
139 indices
[nr
] = iter
.index
;
143 for (i
= 0; i
< nr
; i
++) {
145 radix_tree_delete(root
, index
);
150 static void gmap_rmap_radix_tree_free(struct radix_tree_root
*root
)
152 struct gmap_rmap
*rmap
, *rnext
, *head
;
153 struct radix_tree_iter iter
;
154 unsigned long indices
[16];
159 /* A radix tree is freed by deleting all of its entries */
163 radix_tree_for_each_slot(slot
, root
, &iter
, index
) {
164 indices
[nr
] = iter
.index
;
168 for (i
= 0; i
< nr
; i
++) {
170 head
= radix_tree_delete(root
, index
);
171 gmap_for_each_rmap_safe(rmap
, rnext
, head
)
178 * gmap_free - free a guest address space
179 * @gmap: pointer to the guest address space structure
181 * No locks required. There are no references to this gmap anymore.
183 static void gmap_free(struct gmap
*gmap
)
185 struct page
*page
, *next
;
187 /* Flush tlb of all gmaps (if not already done for shadows) */
188 if (!(gmap_is_shadow(gmap
) && gmap
->removed
))
189 gmap_flush_tlb(gmap
);
190 /* Free all segment & region tables. */
191 list_for_each_entry_safe(page
, next
, &gmap
->crst_list
, lru
)
192 __free_pages(page
, CRST_ALLOC_ORDER
);
193 gmap_radix_tree_free(&gmap
->guest_to_host
);
194 gmap_radix_tree_free(&gmap
->host_to_guest
);
196 /* Free additional data for a shadow gmap */
197 if (gmap_is_shadow(gmap
)) {
198 /* Free all page tables. */
199 list_for_each_entry_safe(page
, next
, &gmap
->pt_list
, lru
)
200 page_table_free_pgste(page
);
201 gmap_rmap_radix_tree_free(&gmap
->host_to_rmap
);
202 /* Release reference to the parent */
203 gmap_put(gmap
->parent
);
210 * gmap_get - increase reference counter for guest address space
211 * @gmap: pointer to the guest address space structure
213 * Returns the gmap pointer
215 struct gmap
*gmap_get(struct gmap
*gmap
)
217 refcount_inc(&gmap
->ref_count
);
220 EXPORT_SYMBOL_GPL(gmap_get
);
223 * gmap_put - decrease reference counter for guest address space
224 * @gmap: pointer to the guest address space structure
226 * If the reference counter reaches zero the guest address space is freed.
228 void gmap_put(struct gmap
*gmap
)
230 if (refcount_dec_and_test(&gmap
->ref_count
))
233 EXPORT_SYMBOL_GPL(gmap_put
);
236 * gmap_remove - remove a guest address space but do not free it yet
237 * @gmap: pointer to the guest address space structure
239 void gmap_remove(struct gmap
*gmap
)
241 struct gmap
*sg
, *next
;
242 unsigned long gmap_asce
;
244 /* Remove all shadow gmaps linked to this gmap */
245 if (!list_empty(&gmap
->children
)) {
246 spin_lock(&gmap
->shadow_lock
);
247 list_for_each_entry_safe(sg
, next
, &gmap
->children
, list
) {
251 spin_unlock(&gmap
->shadow_lock
);
253 /* Remove gmap from the pre-mm list */
254 spin_lock(&gmap
->mm
->context
.lock
);
255 list_del_rcu(&gmap
->list
);
256 if (list_empty(&gmap
->mm
->context
.gmap_list
))
258 else if (list_is_singular(&gmap
->mm
->context
.gmap_list
))
259 gmap_asce
= list_first_entry(&gmap
->mm
->context
.gmap_list
,
260 struct gmap
, list
)->asce
;
263 WRITE_ONCE(gmap
->mm
->context
.gmap_asce
, gmap_asce
);
264 spin_unlock(&gmap
->mm
->context
.lock
);
269 EXPORT_SYMBOL_GPL(gmap_remove
);
272 * gmap_enable - switch primary space to the guest address space
273 * @gmap: pointer to the guest address space structure
275 void gmap_enable(struct gmap
*gmap
)
277 S390_lowcore
.gmap
= (unsigned long) gmap
;
279 EXPORT_SYMBOL_GPL(gmap_enable
);
282 * gmap_disable - switch back to the standard primary address space
283 * @gmap: pointer to the guest address space structure
285 void gmap_disable(struct gmap
*gmap
)
287 S390_lowcore
.gmap
= 0UL;
289 EXPORT_SYMBOL_GPL(gmap_disable
);
292 * gmap_get_enabled - get a pointer to the currently enabled gmap
294 * Returns a pointer to the currently enabled gmap. 0 if none is enabled.
296 struct gmap
*gmap_get_enabled(void)
298 return (struct gmap
*) S390_lowcore
.gmap
;
300 EXPORT_SYMBOL_GPL(gmap_get_enabled
);
303 * gmap_alloc_table is assumed to be called with mmap_sem held
305 static int gmap_alloc_table(struct gmap
*gmap
, unsigned long *table
,
306 unsigned long init
, unsigned long gaddr
)
311 /* since we dont free the gmap table until gmap_free we can unlock */
312 page
= alloc_pages(GFP_KERNEL
, CRST_ALLOC_ORDER
);
315 new = (unsigned long *) page_to_phys(page
);
316 crst_table_init(new, init
);
317 spin_lock(&gmap
->guest_table_lock
);
318 if (*table
& _REGION_ENTRY_INVALID
) {
319 list_add(&page
->lru
, &gmap
->crst_list
);
320 *table
= (unsigned long) new | _REGION_ENTRY_LENGTH
|
321 (*table
& _REGION_ENTRY_TYPE_MASK
);
325 spin_unlock(&gmap
->guest_table_lock
);
327 __free_pages(page
, CRST_ALLOC_ORDER
);
332 * __gmap_segment_gaddr - find virtual address from segment pointer
333 * @entry: pointer to a segment table entry in the guest address space
335 * Returns the virtual address in the guest address space for the segment
337 static unsigned long __gmap_segment_gaddr(unsigned long *entry
)
340 unsigned long offset
, mask
;
342 offset
= (unsigned long) entry
/ sizeof(unsigned long);
343 offset
= (offset
& (PTRS_PER_PMD
- 1)) * PMD_SIZE
;
344 mask
= ~(PTRS_PER_PMD
* sizeof(pmd_t
) - 1);
345 page
= virt_to_page((void *)((unsigned long) entry
& mask
));
346 return page
->index
+ offset
;
350 * __gmap_unlink_by_vmaddr - unlink a single segment via a host address
351 * @gmap: pointer to the guest address space structure
352 * @vmaddr: address in the host process address space
354 * Returns 1 if a TLB flush is required
356 static int __gmap_unlink_by_vmaddr(struct gmap
*gmap
, unsigned long vmaddr
)
358 unsigned long *entry
;
361 BUG_ON(gmap_is_shadow(gmap
));
362 spin_lock(&gmap
->guest_table_lock
);
363 entry
= radix_tree_delete(&gmap
->host_to_guest
, vmaddr
>> PMD_SHIFT
);
365 flush
= (*entry
!= _SEGMENT_ENTRY_EMPTY
);
366 *entry
= _SEGMENT_ENTRY_EMPTY
;
368 spin_unlock(&gmap
->guest_table_lock
);
373 * __gmap_unmap_by_gaddr - unmap a single segment via a guest address
374 * @gmap: pointer to the guest address space structure
375 * @gaddr: address in the guest address space
377 * Returns 1 if a TLB flush is required
379 static int __gmap_unmap_by_gaddr(struct gmap
*gmap
, unsigned long gaddr
)
381 unsigned long vmaddr
;
383 vmaddr
= (unsigned long) radix_tree_delete(&gmap
->guest_to_host
,
385 return vmaddr
? __gmap_unlink_by_vmaddr(gmap
, vmaddr
) : 0;
389 * gmap_unmap_segment - unmap segment from the guest address space
390 * @gmap: pointer to the guest address space structure
391 * @to: address in the guest address space
392 * @len: length of the memory area to unmap
394 * Returns 0 if the unmap succeeded, -EINVAL if not.
396 int gmap_unmap_segment(struct gmap
*gmap
, unsigned long to
, unsigned long len
)
401 BUG_ON(gmap_is_shadow(gmap
));
402 if ((to
| len
) & (PMD_SIZE
- 1))
404 if (len
== 0 || to
+ len
< to
)
408 down_write(&gmap
->mm
->mmap_sem
);
409 for (off
= 0; off
< len
; off
+= PMD_SIZE
)
410 flush
|= __gmap_unmap_by_gaddr(gmap
, to
+ off
);
411 up_write(&gmap
->mm
->mmap_sem
);
413 gmap_flush_tlb(gmap
);
416 EXPORT_SYMBOL_GPL(gmap_unmap_segment
);
419 * gmap_map_segment - map a segment to the guest address space
420 * @gmap: pointer to the guest address space structure
421 * @from: source address in the parent address space
422 * @to: target address in the guest address space
423 * @len: length of the memory area to map
425 * Returns 0 if the mmap succeeded, -EINVAL or -ENOMEM if not.
427 int gmap_map_segment(struct gmap
*gmap
, unsigned long from
,
428 unsigned long to
, unsigned long len
)
433 BUG_ON(gmap_is_shadow(gmap
));
434 if ((from
| to
| len
) & (PMD_SIZE
- 1))
436 if (len
== 0 || from
+ len
< from
|| to
+ len
< to
||
437 from
+ len
- 1 > TASK_SIZE_MAX
|| to
+ len
- 1 > gmap
->asce_end
)
441 down_write(&gmap
->mm
->mmap_sem
);
442 for (off
= 0; off
< len
; off
+= PMD_SIZE
) {
443 /* Remove old translation */
444 flush
|= __gmap_unmap_by_gaddr(gmap
, to
+ off
);
445 /* Store new translation */
446 if (radix_tree_insert(&gmap
->guest_to_host
,
447 (to
+ off
) >> PMD_SHIFT
,
448 (void *) from
+ off
))
451 up_write(&gmap
->mm
->mmap_sem
);
453 gmap_flush_tlb(gmap
);
456 gmap_unmap_segment(gmap
, to
, len
);
459 EXPORT_SYMBOL_GPL(gmap_map_segment
);
462 * __gmap_translate - translate a guest address to a user space address
463 * @gmap: pointer to guest mapping meta data structure
464 * @gaddr: guest address
466 * Returns user space address which corresponds to the guest address or
467 * -EFAULT if no such mapping exists.
468 * This function does not establish potentially missing page table entries.
469 * The mmap_sem of the mm that belongs to the address space must be held
470 * when this function gets called.
472 * Note: Can also be called for shadow gmaps.
474 unsigned long __gmap_translate(struct gmap
*gmap
, unsigned long gaddr
)
476 unsigned long vmaddr
;
478 vmaddr
= (unsigned long)
479 radix_tree_lookup(&gmap
->guest_to_host
, gaddr
>> PMD_SHIFT
);
480 /* Note: guest_to_host is empty for a shadow gmap */
481 return vmaddr
? (vmaddr
| (gaddr
& ~PMD_MASK
)) : -EFAULT
;
483 EXPORT_SYMBOL_GPL(__gmap_translate
);
486 * gmap_translate - translate a guest address to a user space address
487 * @gmap: pointer to guest mapping meta data structure
488 * @gaddr: guest address
490 * Returns user space address which corresponds to the guest address or
491 * -EFAULT if no such mapping exists.
492 * This function does not establish potentially missing page table entries.
494 unsigned long gmap_translate(struct gmap
*gmap
, unsigned long gaddr
)
498 down_read(&gmap
->mm
->mmap_sem
);
499 rc
= __gmap_translate(gmap
, gaddr
);
500 up_read(&gmap
->mm
->mmap_sem
);
503 EXPORT_SYMBOL_GPL(gmap_translate
);
506 * gmap_unlink - disconnect a page table from the gmap shadow tables
507 * @gmap: pointer to guest mapping meta data structure
508 * @table: pointer to the host page table
509 * @vmaddr: vm address associated with the host page table
511 void gmap_unlink(struct mm_struct
*mm
, unsigned long *table
,
512 unsigned long vmaddr
)
518 list_for_each_entry_rcu(gmap
, &mm
->context
.gmap_list
, list
) {
519 flush
= __gmap_unlink_by_vmaddr(gmap
, vmaddr
);
521 gmap_flush_tlb(gmap
);
526 static void gmap_pmdp_xchg(struct gmap
*gmap
, pmd_t
*old
, pmd_t
new,
527 unsigned long gaddr
);
530 * gmap_link - set up shadow page tables to connect a host to a guest address
531 * @gmap: pointer to guest mapping meta data structure
532 * @gaddr: guest address
533 * @vmaddr: vm address
535 * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT
536 * if the vm address is already mapped to a different guest segment.
537 * The mmap_sem of the mm that belongs to the address space must be held
538 * when this function gets called.
540 int __gmap_link(struct gmap
*gmap
, unsigned long gaddr
, unsigned long vmaddr
)
542 struct mm_struct
*mm
;
543 unsigned long *table
;
552 BUG_ON(gmap_is_shadow(gmap
));
553 /* Create higher level tables in the gmap page table */
555 if ((gmap
->asce
& _ASCE_TYPE_MASK
) >= _ASCE_TYPE_REGION1
) {
556 table
+= (gaddr
& _REGION1_INDEX
) >> _REGION1_SHIFT
;
557 if ((*table
& _REGION_ENTRY_INVALID
) &&
558 gmap_alloc_table(gmap
, table
, _REGION2_ENTRY_EMPTY
,
559 gaddr
& _REGION1_MASK
))
561 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
563 if ((gmap
->asce
& _ASCE_TYPE_MASK
) >= _ASCE_TYPE_REGION2
) {
564 table
+= (gaddr
& _REGION2_INDEX
) >> _REGION2_SHIFT
;
565 if ((*table
& _REGION_ENTRY_INVALID
) &&
566 gmap_alloc_table(gmap
, table
, _REGION3_ENTRY_EMPTY
,
567 gaddr
& _REGION2_MASK
))
569 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
571 if ((gmap
->asce
& _ASCE_TYPE_MASK
) >= _ASCE_TYPE_REGION3
) {
572 table
+= (gaddr
& _REGION3_INDEX
) >> _REGION3_SHIFT
;
573 if ((*table
& _REGION_ENTRY_INVALID
) &&
574 gmap_alloc_table(gmap
, table
, _SEGMENT_ENTRY_EMPTY
,
575 gaddr
& _REGION3_MASK
))
577 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
579 table
+= (gaddr
& _SEGMENT_INDEX
) >> _SEGMENT_SHIFT
;
580 /* Walk the parent mm page table */
582 pgd
= pgd_offset(mm
, vmaddr
);
583 VM_BUG_ON(pgd_none(*pgd
));
584 p4d
= p4d_offset(pgd
, vmaddr
);
585 VM_BUG_ON(p4d_none(*p4d
));
586 pud
= pud_offset(p4d
, vmaddr
);
587 VM_BUG_ON(pud_none(*pud
));
588 /* large puds cannot yet be handled */
591 pmd
= pmd_offset(pud
, vmaddr
);
592 VM_BUG_ON(pmd_none(*pmd
));
593 /* Are we allowed to use huge pages? */
594 if (pmd_large(*pmd
) && !gmap
->mm
->context
.allow_gmap_hpage_1m
)
596 /* Link gmap segment table entry location to page table. */
597 rc
= radix_tree_preload(GFP_KERNEL
);
600 ptl
= pmd_lock(mm
, pmd
);
601 spin_lock(&gmap
->guest_table_lock
);
602 if (*table
== _SEGMENT_ENTRY_EMPTY
) {
603 rc
= radix_tree_insert(&gmap
->host_to_guest
,
604 vmaddr
>> PMD_SHIFT
, table
);
606 if (pmd_large(*pmd
)) {
607 *table
= (pmd_val(*pmd
) &
608 _SEGMENT_ENTRY_HARDWARE_BITS_LARGE
)
609 | _SEGMENT_ENTRY_GMAP_UC
;
611 *table
= pmd_val(*pmd
) &
612 _SEGMENT_ENTRY_HARDWARE_BITS
;
614 } else if (*table
& _SEGMENT_ENTRY_PROTECT
&&
615 !(pmd_val(*pmd
) & _SEGMENT_ENTRY_PROTECT
)) {
616 unprot
= (u64
)*table
;
617 unprot
&= ~_SEGMENT_ENTRY_PROTECT
;
618 unprot
|= _SEGMENT_ENTRY_GMAP_UC
;
619 gmap_pmdp_xchg(gmap
, (pmd_t
*)table
, __pmd(unprot
), gaddr
);
621 spin_unlock(&gmap
->guest_table_lock
);
623 radix_tree_preload_end();
628 * gmap_fault - resolve a fault on a guest address
629 * @gmap: pointer to guest mapping meta data structure
630 * @gaddr: guest address
631 * @fault_flags: flags to pass down to handle_mm_fault()
633 * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT
634 * if the vm address is already mapped to a different guest segment.
636 int gmap_fault(struct gmap
*gmap
, unsigned long gaddr
,
637 unsigned int fault_flags
)
639 unsigned long vmaddr
;
643 down_read(&gmap
->mm
->mmap_sem
);
647 vmaddr
= __gmap_translate(gmap
, gaddr
);
648 if (IS_ERR_VALUE(vmaddr
)) {
652 if (fixup_user_fault(current
, gmap
->mm
, vmaddr
, fault_flags
,
658 * In the case that fixup_user_fault unlocked the mmap_sem during
659 * faultin redo __gmap_translate to not race with a map/unmap_segment.
664 rc
= __gmap_link(gmap
, gaddr
, vmaddr
);
666 up_read(&gmap
->mm
->mmap_sem
);
669 EXPORT_SYMBOL_GPL(gmap_fault
);
672 * this function is assumed to be called with mmap_sem held
674 void __gmap_zap(struct gmap
*gmap
, unsigned long gaddr
)
676 unsigned long vmaddr
;
680 /* Find the vm address for the guest address */
681 vmaddr
= (unsigned long) radix_tree_lookup(&gmap
->guest_to_host
,
684 vmaddr
|= gaddr
& ~PMD_MASK
;
685 /* Get pointer to the page table entry */
686 ptep
= get_locked_pte(gmap
->mm
, vmaddr
, &ptl
);
688 ptep_zap_unused(gmap
->mm
, vmaddr
, ptep
, 0);
689 pte_unmap_unlock(ptep
, ptl
);
692 EXPORT_SYMBOL_GPL(__gmap_zap
);
694 void gmap_discard(struct gmap
*gmap
, unsigned long from
, unsigned long to
)
696 unsigned long gaddr
, vmaddr
, size
;
697 struct vm_area_struct
*vma
;
699 down_read(&gmap
->mm
->mmap_sem
);
700 for (gaddr
= from
; gaddr
< to
;
701 gaddr
= (gaddr
+ PMD_SIZE
) & PMD_MASK
) {
702 /* Find the vm address for the guest address */
703 vmaddr
= (unsigned long)
704 radix_tree_lookup(&gmap
->guest_to_host
,
708 vmaddr
|= gaddr
& ~PMD_MASK
;
709 /* Find vma in the parent mm */
710 vma
= find_vma(gmap
->mm
, vmaddr
);
714 * We do not discard pages that are backed by
715 * hugetlbfs, so we don't have to refault them.
717 if (is_vm_hugetlb_page(vma
))
719 size
= min(to
- gaddr
, PMD_SIZE
- (gaddr
& ~PMD_MASK
));
720 zap_page_range(vma
, vmaddr
, size
);
722 up_read(&gmap
->mm
->mmap_sem
);
724 EXPORT_SYMBOL_GPL(gmap_discard
);
726 static LIST_HEAD(gmap_notifier_list
);
727 static DEFINE_SPINLOCK(gmap_notifier_lock
);
730 * gmap_register_pte_notifier - register a pte invalidation callback
731 * @nb: pointer to the gmap notifier block
733 void gmap_register_pte_notifier(struct gmap_notifier
*nb
)
735 spin_lock(&gmap_notifier_lock
);
736 list_add_rcu(&nb
->list
, &gmap_notifier_list
);
737 spin_unlock(&gmap_notifier_lock
);
739 EXPORT_SYMBOL_GPL(gmap_register_pte_notifier
);
742 * gmap_unregister_pte_notifier - remove a pte invalidation callback
743 * @nb: pointer to the gmap notifier block
745 void gmap_unregister_pte_notifier(struct gmap_notifier
*nb
)
747 spin_lock(&gmap_notifier_lock
);
748 list_del_rcu(&nb
->list
);
749 spin_unlock(&gmap_notifier_lock
);
752 EXPORT_SYMBOL_GPL(gmap_unregister_pte_notifier
);
755 * gmap_call_notifier - call all registered invalidation callbacks
756 * @gmap: pointer to guest mapping meta data structure
757 * @start: start virtual address in the guest address space
758 * @end: end virtual address in the guest address space
760 static void gmap_call_notifier(struct gmap
*gmap
, unsigned long start
,
763 struct gmap_notifier
*nb
;
765 list_for_each_entry(nb
, &gmap_notifier_list
, list
)
766 nb
->notifier_call(gmap
, start
, end
);
770 * gmap_table_walk - walk the gmap page tables
771 * @gmap: pointer to guest mapping meta data structure
772 * @gaddr: virtual address in the guest address space
773 * @level: page table level to stop at
775 * Returns a table entry pointer for the given guest address and @level
776 * @level=0 : returns a pointer to a page table table entry (or NULL)
777 * @level=1 : returns a pointer to a segment table entry (or NULL)
778 * @level=2 : returns a pointer to a region-3 table entry (or NULL)
779 * @level=3 : returns a pointer to a region-2 table entry (or NULL)
780 * @level=4 : returns a pointer to a region-1 table entry (or NULL)
782 * Returns NULL if the gmap page tables could not be walked to the
785 * Note: Can also be called for shadow gmaps.
787 static inline unsigned long *gmap_table_walk(struct gmap
*gmap
,
788 unsigned long gaddr
, int level
)
790 unsigned long *table
;
792 if ((gmap
->asce
& _ASCE_TYPE_MASK
) + 4 < (level
* 4))
794 if (gmap_is_shadow(gmap
) && gmap
->removed
)
796 if (gaddr
& (-1UL << (31 + ((gmap
->asce
& _ASCE_TYPE_MASK
) >> 2)*11)))
799 switch (gmap
->asce
& _ASCE_TYPE_MASK
) {
800 case _ASCE_TYPE_REGION1
:
801 table
+= (gaddr
& _REGION1_INDEX
) >> _REGION1_SHIFT
;
804 if (*table
& _REGION_ENTRY_INVALID
)
806 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
808 case _ASCE_TYPE_REGION2
:
809 table
+= (gaddr
& _REGION2_INDEX
) >> _REGION2_SHIFT
;
812 if (*table
& _REGION_ENTRY_INVALID
)
814 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
816 case _ASCE_TYPE_REGION3
:
817 table
+= (gaddr
& _REGION3_INDEX
) >> _REGION3_SHIFT
;
820 if (*table
& _REGION_ENTRY_INVALID
)
822 table
= (unsigned long *)(*table
& _REGION_ENTRY_ORIGIN
);
824 case _ASCE_TYPE_SEGMENT
:
825 table
+= (gaddr
& _SEGMENT_INDEX
) >> _SEGMENT_SHIFT
;
828 if (*table
& _REGION_ENTRY_INVALID
)
830 table
= (unsigned long *)(*table
& _SEGMENT_ENTRY_ORIGIN
);
831 table
+= (gaddr
& _PAGE_INDEX
) >> _PAGE_SHIFT
;
837 * gmap_pte_op_walk - walk the gmap page table, get the page table lock
838 * and return the pte pointer
839 * @gmap: pointer to guest mapping meta data structure
840 * @gaddr: virtual address in the guest address space
841 * @ptl: pointer to the spinlock pointer
843 * Returns a pointer to the locked pte for a guest address, or NULL
845 static pte_t
*gmap_pte_op_walk(struct gmap
*gmap
, unsigned long gaddr
,
848 unsigned long *table
;
850 BUG_ON(gmap_is_shadow(gmap
));
851 /* Walk the gmap page table, lock and get pte pointer */
852 table
= gmap_table_walk(gmap
, gaddr
, 1); /* get segment pointer */
853 if (!table
|| *table
& _SEGMENT_ENTRY_INVALID
)
855 return pte_alloc_map_lock(gmap
->mm
, (pmd_t
*) table
, gaddr
, ptl
);
859 * gmap_pte_op_fixup - force a page in and connect the gmap page table
860 * @gmap: pointer to guest mapping meta data structure
861 * @gaddr: virtual address in the guest address space
862 * @vmaddr: address in the host process address space
863 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
865 * Returns 0 if the caller can retry __gmap_translate (might fail again),
866 * -ENOMEM if out of memory and -EFAULT if anything goes wrong while fixing
867 * up or connecting the gmap page table.
869 static int gmap_pte_op_fixup(struct gmap
*gmap
, unsigned long gaddr
,
870 unsigned long vmaddr
, int prot
)
872 struct mm_struct
*mm
= gmap
->mm
;
873 unsigned int fault_flags
;
874 bool unlocked
= false;
876 BUG_ON(gmap_is_shadow(gmap
));
877 fault_flags
= (prot
== PROT_WRITE
) ? FAULT_FLAG_WRITE
: 0;
878 if (fixup_user_fault(current
, mm
, vmaddr
, fault_flags
, &unlocked
))
881 /* lost mmap_sem, caller has to retry __gmap_translate */
883 /* Connect the page tables */
884 return __gmap_link(gmap
, gaddr
, vmaddr
);
888 * gmap_pte_op_end - release the page table lock
889 * @ptl: pointer to the spinlock pointer
891 static void gmap_pte_op_end(spinlock_t
*ptl
)
898 * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
899 * and return the pmd pointer
900 * @gmap: pointer to guest mapping meta data structure
901 * @gaddr: virtual address in the guest address space
903 * Returns a pointer to the pmd for a guest address, or NULL
905 static inline pmd_t
*gmap_pmd_op_walk(struct gmap
*gmap
, unsigned long gaddr
)
909 BUG_ON(gmap_is_shadow(gmap
));
910 pmdp
= (pmd_t
*) gmap_table_walk(gmap
, gaddr
, 1);
914 /* without huge pages, there is no need to take the table lock */
915 if (!gmap
->mm
->context
.allow_gmap_hpage_1m
)
916 return pmd_none(*pmdp
) ? NULL
: pmdp
;
918 spin_lock(&gmap
->guest_table_lock
);
919 if (pmd_none(*pmdp
)) {
920 spin_unlock(&gmap
->guest_table_lock
);
924 /* 4k page table entries are locked via the pte (pte_alloc_map_lock). */
925 if (!pmd_large(*pmdp
))
926 spin_unlock(&gmap
->guest_table_lock
);
931 * gmap_pmd_op_end - release the guest_table_lock if needed
932 * @gmap: pointer to the guest mapping meta data structure
933 * @pmdp: pointer to the pmd
935 static inline void gmap_pmd_op_end(struct gmap
*gmap
, pmd_t
*pmdp
)
937 if (pmd_large(*pmdp
))
938 spin_unlock(&gmap
->guest_table_lock
);
942 * gmap_protect_pmd - remove access rights to memory and set pmd notification bits
943 * @pmdp: pointer to the pmd to be protected
944 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
945 * @bits: notification bits to set
948 * 0 if successfully protected
949 * -EAGAIN if a fixup is needed
950 * -EINVAL if unsupported notifier bits have been specified
952 * Expected to be called with sg->mm->mmap_sem in read and
953 * guest_table_lock held.
955 static int gmap_protect_pmd(struct gmap
*gmap
, unsigned long gaddr
,
956 pmd_t
*pmdp
, int prot
, unsigned long bits
)
958 int pmd_i
= pmd_val(*pmdp
) & _SEGMENT_ENTRY_INVALID
;
959 int pmd_p
= pmd_val(*pmdp
) & _SEGMENT_ENTRY_PROTECT
;
963 if ((pmd_i
&& (prot
!= PROT_NONE
)) || (pmd_p
&& (prot
== PROT_WRITE
)))
966 if (prot
== PROT_NONE
&& !pmd_i
) {
967 pmd_val(new) |= _SEGMENT_ENTRY_INVALID
;
968 gmap_pmdp_xchg(gmap
, pmdp
, new, gaddr
);
971 if (prot
== PROT_READ
&& !pmd_p
) {
972 pmd_val(new) &= ~_SEGMENT_ENTRY_INVALID
;
973 pmd_val(new) |= _SEGMENT_ENTRY_PROTECT
;
974 gmap_pmdp_xchg(gmap
, pmdp
, new, gaddr
);
977 if (bits
& GMAP_NOTIFY_MPROT
)
978 pmd_val(*pmdp
) |= _SEGMENT_ENTRY_GMAP_IN
;
980 /* Shadow GMAP protection needs split PMDs */
981 if (bits
& GMAP_NOTIFY_SHADOW
)
988 * gmap_protect_pte - remove access rights to memory and set pgste bits
989 * @gmap: pointer to guest mapping meta data structure
990 * @gaddr: virtual address in the guest address space
991 * @pmdp: pointer to the pmd associated with the pte
992 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
993 * @bits: notification bits to set
995 * Returns 0 if successfully protected, -ENOMEM if out of memory and
996 * -EAGAIN if a fixup is needed.
998 * Expected to be called with sg->mm->mmap_sem in read
1000 static int gmap_protect_pte(struct gmap
*gmap
, unsigned long gaddr
,
1001 pmd_t
*pmdp
, int prot
, unsigned long bits
)
1005 spinlock_t
*ptl
= NULL
;
1006 unsigned long pbits
= 0;
1008 if (pmd_val(*pmdp
) & _SEGMENT_ENTRY_INVALID
)
1011 ptep
= pte_alloc_map_lock(gmap
->mm
, pmdp
, gaddr
, &ptl
);
1015 pbits
|= (bits
& GMAP_NOTIFY_MPROT
) ? PGSTE_IN_BIT
: 0;
1016 pbits
|= (bits
& GMAP_NOTIFY_SHADOW
) ? PGSTE_VSIE_BIT
: 0;
1017 /* Protect and unlock. */
1018 rc
= ptep_force_prot(gmap
->mm
, gaddr
, ptep
, prot
, pbits
);
1019 gmap_pte_op_end(ptl
);
1024 * gmap_protect_range - remove access rights to memory and set pgste bits
1025 * @gmap: pointer to guest mapping meta data structure
1026 * @gaddr: virtual address in the guest address space
1027 * @len: size of area
1028 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
1029 * @bits: pgste notification bits to set
1031 * Returns 0 if successfully protected, -ENOMEM if out of memory and
1032 * -EFAULT if gaddr is invalid (or mapping for shadows is missing).
1034 * Called with sg->mm->mmap_sem in read.
1036 static int gmap_protect_range(struct gmap
*gmap
, unsigned long gaddr
,
1037 unsigned long len
, int prot
, unsigned long bits
)
1039 unsigned long vmaddr
, dist
;
1043 BUG_ON(gmap_is_shadow(gmap
));
1046 pmdp
= gmap_pmd_op_walk(gmap
, gaddr
);
1048 if (!pmd_large(*pmdp
)) {
1049 rc
= gmap_protect_pte(gmap
, gaddr
, pmdp
, prot
,
1056 rc
= gmap_protect_pmd(gmap
, gaddr
, pmdp
, prot
,
1059 dist
= HPAGE_SIZE
- (gaddr
& ~HPAGE_MASK
);
1060 len
= len
< dist
? 0 : len
- dist
;
1061 gaddr
= (gaddr
& HPAGE_MASK
) + HPAGE_SIZE
;
1064 gmap_pmd_op_end(gmap
, pmdp
);
1070 /* -EAGAIN, fixup of userspace mm and gmap */
1071 vmaddr
= __gmap_translate(gmap
, gaddr
);
1072 if (IS_ERR_VALUE(vmaddr
))
1074 rc
= gmap_pte_op_fixup(gmap
, gaddr
, vmaddr
, prot
);
1083 * gmap_mprotect_notify - change access rights for a range of ptes and
1084 * call the notifier if any pte changes again
1085 * @gmap: pointer to guest mapping meta data structure
1086 * @gaddr: virtual address in the guest address space
1087 * @len: size of area
1088 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
1090 * Returns 0 if for each page in the given range a gmap mapping exists,
1091 * the new access rights could be set and the notifier could be armed.
1092 * If the gmap mapping is missing for one or more pages -EFAULT is
1093 * returned. If no memory could be allocated -ENOMEM is returned.
1094 * This function establishes missing page table entries.
1096 int gmap_mprotect_notify(struct gmap
*gmap
, unsigned long gaddr
,
1097 unsigned long len
, int prot
)
1101 if ((gaddr
& ~PAGE_MASK
) || (len
& ~PAGE_MASK
) || gmap_is_shadow(gmap
))
1103 if (!MACHINE_HAS_ESOP
&& prot
== PROT_READ
)
1105 down_read(&gmap
->mm
->mmap_sem
);
1106 rc
= gmap_protect_range(gmap
, gaddr
, len
, prot
, GMAP_NOTIFY_MPROT
);
1107 up_read(&gmap
->mm
->mmap_sem
);
1110 EXPORT_SYMBOL_GPL(gmap_mprotect_notify
);
1113 * gmap_read_table - get an unsigned long value from a guest page table using
1114 * absolute addressing, without marking the page referenced.
1115 * @gmap: pointer to guest mapping meta data structure
1116 * @gaddr: virtual address in the guest address space
1117 * @val: pointer to the unsigned long value to return
1119 * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT
1120 * if reading using the virtual address failed. -EINVAL if called on a gmap
1123 * Called with gmap->mm->mmap_sem in read.
1125 int gmap_read_table(struct gmap
*gmap
, unsigned long gaddr
, unsigned long *val
)
1127 unsigned long address
, vmaddr
;
1132 if (gmap_is_shadow(gmap
))
1137 ptep
= gmap_pte_op_walk(gmap
, gaddr
, &ptl
);
1140 if (pte_present(pte
) && (pte_val(pte
) & _PAGE_READ
)) {
1141 address
= pte_val(pte
) & PAGE_MASK
;
1142 address
+= gaddr
& ~PAGE_MASK
;
1143 *val
= *(unsigned long *) address
;
1144 pte_val(*ptep
) |= _PAGE_YOUNG
;
1145 /* Do *NOT* clear the _PAGE_INVALID bit! */
1148 gmap_pte_op_end(ptl
);
1152 vmaddr
= __gmap_translate(gmap
, gaddr
);
1153 if (IS_ERR_VALUE(vmaddr
)) {
1157 rc
= gmap_pte_op_fixup(gmap
, gaddr
, vmaddr
, PROT_READ
);
1163 EXPORT_SYMBOL_GPL(gmap_read_table
);
1166 * gmap_insert_rmap - add a rmap to the host_to_rmap radix tree
1167 * @sg: pointer to the shadow guest address space structure
1168 * @vmaddr: vm address associated with the rmap
1169 * @rmap: pointer to the rmap structure
1171 * Called with the sg->guest_table_lock
1173 static inline void gmap_insert_rmap(struct gmap
*sg
, unsigned long vmaddr
,
1174 struct gmap_rmap
*rmap
)
1178 BUG_ON(!gmap_is_shadow(sg
));
1179 slot
= radix_tree_lookup_slot(&sg
->host_to_rmap
, vmaddr
>> PAGE_SHIFT
);
1181 rmap
->next
= radix_tree_deref_slot_protected(slot
,
1182 &sg
->guest_table_lock
);
1183 radix_tree_replace_slot(&sg
->host_to_rmap
, slot
, rmap
);
1186 radix_tree_insert(&sg
->host_to_rmap
, vmaddr
>> PAGE_SHIFT
,
1192 * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap
1193 * @sg: pointer to the shadow guest address space structure
1194 * @raddr: rmap address in the shadow gmap
1195 * @paddr: address in the parent guest address space
1196 * @len: length of the memory area to protect
1198 * Returns 0 if successfully protected and the rmap was created, -ENOMEM
1199 * if out of memory and -EFAULT if paddr is invalid.
1201 static int gmap_protect_rmap(struct gmap
*sg
, unsigned long raddr
,
1202 unsigned long paddr
, unsigned long len
)
1204 struct gmap
*parent
;
1205 struct gmap_rmap
*rmap
;
1206 unsigned long vmaddr
;
1211 BUG_ON(!gmap_is_shadow(sg
));
1212 parent
= sg
->parent
;
1214 vmaddr
= __gmap_translate(parent
, paddr
);
1215 if (IS_ERR_VALUE(vmaddr
))
1217 rmap
= kzalloc(sizeof(*rmap
), GFP_KERNEL
);
1220 rmap
->raddr
= raddr
;
1221 rc
= radix_tree_preload(GFP_KERNEL
);
1227 ptep
= gmap_pte_op_walk(parent
, paddr
, &ptl
);
1229 spin_lock(&sg
->guest_table_lock
);
1230 rc
= ptep_force_prot(parent
->mm
, paddr
, ptep
, PROT_READ
,
1233 gmap_insert_rmap(sg
, vmaddr
, rmap
);
1234 spin_unlock(&sg
->guest_table_lock
);
1235 gmap_pte_op_end(ptl
);
1237 radix_tree_preload_end();
1240 rc
= gmap_pte_op_fixup(parent
, paddr
, vmaddr
, PROT_READ
);
1251 #define _SHADOW_RMAP_MASK 0x7
1252 #define _SHADOW_RMAP_REGION1 0x5
1253 #define _SHADOW_RMAP_REGION2 0x4
1254 #define _SHADOW_RMAP_REGION3 0x3
1255 #define _SHADOW_RMAP_SEGMENT 0x2
1256 #define _SHADOW_RMAP_PGTABLE 0x1
1259 * gmap_idte_one - invalidate a single region or segment table entry
1260 * @asce: region or segment table *origin* + table-type bits
1261 * @vaddr: virtual address to identify the table entry to flush
1263 * The invalid bit of a single region or segment table entry is set
1264 * and the associated TLB entries depending on the entry are flushed.
1265 * The table-type of the @asce identifies the portion of the @vaddr
1266 * that is used as the invalidation index.
1268 static inline void gmap_idte_one(unsigned long asce
, unsigned long vaddr
)
1271 " .insn rrf,0xb98e0000,%0,%1,0,0"
1272 : : "a" (asce
), "a" (vaddr
) : "cc", "memory");
1276 * gmap_unshadow_page - remove a page from a shadow page table
1277 * @sg: pointer to the shadow guest address space structure
1278 * @raddr: rmap address in the shadow guest address space
1280 * Called with the sg->guest_table_lock
1282 static void gmap_unshadow_page(struct gmap
*sg
, unsigned long raddr
)
1284 unsigned long *table
;
1286 BUG_ON(!gmap_is_shadow(sg
));
1287 table
= gmap_table_walk(sg
, raddr
, 0); /* get page table pointer */
1288 if (!table
|| *table
& _PAGE_INVALID
)
1290 gmap_call_notifier(sg
, raddr
, raddr
+ _PAGE_SIZE
- 1);
1291 ptep_unshadow_pte(sg
->mm
, raddr
, (pte_t
*) table
);
1295 * __gmap_unshadow_pgt - remove all entries from a shadow page table
1296 * @sg: pointer to the shadow guest address space structure
1297 * @raddr: rmap address in the shadow guest address space
1298 * @pgt: pointer to the start of a shadow page table
1300 * Called with the sg->guest_table_lock
1302 static void __gmap_unshadow_pgt(struct gmap
*sg
, unsigned long raddr
,
1307 BUG_ON(!gmap_is_shadow(sg
));
1308 for (i
= 0; i
< _PAGE_ENTRIES
; i
++, raddr
+= _PAGE_SIZE
)
1309 pgt
[i
] = _PAGE_INVALID
;
1313 * gmap_unshadow_pgt - remove a shadow page table from a segment entry
1314 * @sg: pointer to the shadow guest address space structure
1315 * @raddr: address in the shadow guest address space
1317 * Called with the sg->guest_table_lock
1319 static void gmap_unshadow_pgt(struct gmap
*sg
, unsigned long raddr
)
1321 unsigned long sto
, *ste
, *pgt
;
1324 BUG_ON(!gmap_is_shadow(sg
));
1325 ste
= gmap_table_walk(sg
, raddr
, 1); /* get segment pointer */
1326 if (!ste
|| !(*ste
& _SEGMENT_ENTRY_ORIGIN
))
1328 gmap_call_notifier(sg
, raddr
, raddr
+ _SEGMENT_SIZE
- 1);
1329 sto
= (unsigned long) (ste
- ((raddr
& _SEGMENT_INDEX
) >> _SEGMENT_SHIFT
));
1330 gmap_idte_one(sto
| _ASCE_TYPE_SEGMENT
, raddr
);
1331 pgt
= (unsigned long *)(*ste
& _SEGMENT_ENTRY_ORIGIN
);
1332 *ste
= _SEGMENT_ENTRY_EMPTY
;
1333 __gmap_unshadow_pgt(sg
, raddr
, pgt
);
1334 /* Free page table */
1335 page
= pfn_to_page(__pa(pgt
) >> PAGE_SHIFT
);
1336 list_del(&page
->lru
);
1337 page_table_free_pgste(page
);
1341 * __gmap_unshadow_sgt - remove all entries from a shadow segment table
1342 * @sg: pointer to the shadow guest address space structure
1343 * @raddr: rmap address in the shadow guest address space
1344 * @sgt: pointer to the start of a shadow segment table
1346 * Called with the sg->guest_table_lock
1348 static void __gmap_unshadow_sgt(struct gmap
*sg
, unsigned long raddr
,
1355 BUG_ON(!gmap_is_shadow(sg
));
1356 for (i
= 0; i
< _CRST_ENTRIES
; i
++, raddr
+= _SEGMENT_SIZE
) {
1357 if (!(sgt
[i
] & _SEGMENT_ENTRY_ORIGIN
))
1359 pgt
= (unsigned long *)(sgt
[i
] & _REGION_ENTRY_ORIGIN
);
1360 sgt
[i
] = _SEGMENT_ENTRY_EMPTY
;
1361 __gmap_unshadow_pgt(sg
, raddr
, pgt
);
1362 /* Free page table */
1363 page
= pfn_to_page(__pa(pgt
) >> PAGE_SHIFT
);
1364 list_del(&page
->lru
);
1365 page_table_free_pgste(page
);
1370 * gmap_unshadow_sgt - remove a shadow segment table from a region-3 entry
1371 * @sg: pointer to the shadow guest address space structure
1372 * @raddr: rmap address in the shadow guest address space
1374 * Called with the shadow->guest_table_lock
1376 static void gmap_unshadow_sgt(struct gmap
*sg
, unsigned long raddr
)
1378 unsigned long r3o
, *r3e
, *sgt
;
1381 BUG_ON(!gmap_is_shadow(sg
));
1382 r3e
= gmap_table_walk(sg
, raddr
, 2); /* get region-3 pointer */
1383 if (!r3e
|| !(*r3e
& _REGION_ENTRY_ORIGIN
))
1385 gmap_call_notifier(sg
, raddr
, raddr
+ _REGION3_SIZE
- 1);
1386 r3o
= (unsigned long) (r3e
- ((raddr
& _REGION3_INDEX
) >> _REGION3_SHIFT
));
1387 gmap_idte_one(r3o
| _ASCE_TYPE_REGION3
, raddr
);
1388 sgt
= (unsigned long *)(*r3e
& _REGION_ENTRY_ORIGIN
);
1389 *r3e
= _REGION3_ENTRY_EMPTY
;
1390 __gmap_unshadow_sgt(sg
, raddr
, sgt
);
1391 /* Free segment table */
1392 page
= pfn_to_page(__pa(sgt
) >> PAGE_SHIFT
);
1393 list_del(&page
->lru
);
1394 __free_pages(page
, CRST_ALLOC_ORDER
);
1398 * __gmap_unshadow_r3t - remove all entries from a shadow region-3 table
1399 * @sg: pointer to the shadow guest address space structure
1400 * @raddr: address in the shadow guest address space
1401 * @r3t: pointer to the start of a shadow region-3 table
1403 * Called with the sg->guest_table_lock
1405 static void __gmap_unshadow_r3t(struct gmap
*sg
, unsigned long raddr
,
1412 BUG_ON(!gmap_is_shadow(sg
));
1413 for (i
= 0; i
< _CRST_ENTRIES
; i
++, raddr
+= _REGION3_SIZE
) {
1414 if (!(r3t
[i
] & _REGION_ENTRY_ORIGIN
))
1416 sgt
= (unsigned long *)(r3t
[i
] & _REGION_ENTRY_ORIGIN
);
1417 r3t
[i
] = _REGION3_ENTRY_EMPTY
;
1418 __gmap_unshadow_sgt(sg
, raddr
, sgt
);
1419 /* Free segment table */
1420 page
= pfn_to_page(__pa(sgt
) >> PAGE_SHIFT
);
1421 list_del(&page
->lru
);
1422 __free_pages(page
, CRST_ALLOC_ORDER
);
1427 * gmap_unshadow_r3t - remove a shadow region-3 table from a region-2 entry
1428 * @sg: pointer to the shadow guest address space structure
1429 * @raddr: rmap address in the shadow guest address space
1431 * Called with the sg->guest_table_lock
1433 static void gmap_unshadow_r3t(struct gmap
*sg
, unsigned long raddr
)
1435 unsigned long r2o
, *r2e
, *r3t
;
1438 BUG_ON(!gmap_is_shadow(sg
));
1439 r2e
= gmap_table_walk(sg
, raddr
, 3); /* get region-2 pointer */
1440 if (!r2e
|| !(*r2e
& _REGION_ENTRY_ORIGIN
))
1442 gmap_call_notifier(sg
, raddr
, raddr
+ _REGION2_SIZE
- 1);
1443 r2o
= (unsigned long) (r2e
- ((raddr
& _REGION2_INDEX
) >> _REGION2_SHIFT
));
1444 gmap_idte_one(r2o
| _ASCE_TYPE_REGION2
, raddr
);
1445 r3t
= (unsigned long *)(*r2e
& _REGION_ENTRY_ORIGIN
);
1446 *r2e
= _REGION2_ENTRY_EMPTY
;
1447 __gmap_unshadow_r3t(sg
, raddr
, r3t
);
1448 /* Free region 3 table */
1449 page
= pfn_to_page(__pa(r3t
) >> PAGE_SHIFT
);
1450 list_del(&page
->lru
);
1451 __free_pages(page
, CRST_ALLOC_ORDER
);
1455 * __gmap_unshadow_r2t - remove all entries from a shadow region-2 table
1456 * @sg: pointer to the shadow guest address space structure
1457 * @raddr: rmap address in the shadow guest address space
1458 * @r2t: pointer to the start of a shadow region-2 table
1460 * Called with the sg->guest_table_lock
1462 static void __gmap_unshadow_r2t(struct gmap
*sg
, unsigned long raddr
,
1469 BUG_ON(!gmap_is_shadow(sg
));
1470 for (i
= 0; i
< _CRST_ENTRIES
; i
++, raddr
+= _REGION2_SIZE
) {
1471 if (!(r2t
[i
] & _REGION_ENTRY_ORIGIN
))
1473 r3t
= (unsigned long *)(r2t
[i
] & _REGION_ENTRY_ORIGIN
);
1474 r2t
[i
] = _REGION2_ENTRY_EMPTY
;
1475 __gmap_unshadow_r3t(sg
, raddr
, r3t
);
1476 /* Free region 3 table */
1477 page
= pfn_to_page(__pa(r3t
) >> PAGE_SHIFT
);
1478 list_del(&page
->lru
);
1479 __free_pages(page
, CRST_ALLOC_ORDER
);
1484 * gmap_unshadow_r2t - remove a shadow region-2 table from a region-1 entry
1485 * @sg: pointer to the shadow guest address space structure
1486 * @raddr: rmap address in the shadow guest address space
1488 * Called with the sg->guest_table_lock
1490 static void gmap_unshadow_r2t(struct gmap
*sg
, unsigned long raddr
)
1492 unsigned long r1o
, *r1e
, *r2t
;
1495 BUG_ON(!gmap_is_shadow(sg
));
1496 r1e
= gmap_table_walk(sg
, raddr
, 4); /* get region-1 pointer */
1497 if (!r1e
|| !(*r1e
& _REGION_ENTRY_ORIGIN
))
1499 gmap_call_notifier(sg
, raddr
, raddr
+ _REGION1_SIZE
- 1);
1500 r1o
= (unsigned long) (r1e
- ((raddr
& _REGION1_INDEX
) >> _REGION1_SHIFT
));
1501 gmap_idte_one(r1o
| _ASCE_TYPE_REGION1
, raddr
);
1502 r2t
= (unsigned long *)(*r1e
& _REGION_ENTRY_ORIGIN
);
1503 *r1e
= _REGION1_ENTRY_EMPTY
;
1504 __gmap_unshadow_r2t(sg
, raddr
, r2t
);
1505 /* Free region 2 table */
1506 page
= pfn_to_page(__pa(r2t
) >> PAGE_SHIFT
);
1507 list_del(&page
->lru
);
1508 __free_pages(page
, CRST_ALLOC_ORDER
);
1512 * __gmap_unshadow_r1t - remove all entries from a shadow region-1 table
1513 * @sg: pointer to the shadow guest address space structure
1514 * @raddr: rmap address in the shadow guest address space
1515 * @r1t: pointer to the start of a shadow region-1 table
1517 * Called with the shadow->guest_table_lock
1519 static void __gmap_unshadow_r1t(struct gmap
*sg
, unsigned long raddr
,
1522 unsigned long asce
, *r2t
;
1526 BUG_ON(!gmap_is_shadow(sg
));
1527 asce
= (unsigned long) r1t
| _ASCE_TYPE_REGION1
;
1528 for (i
= 0; i
< _CRST_ENTRIES
; i
++, raddr
+= _REGION1_SIZE
) {
1529 if (!(r1t
[i
] & _REGION_ENTRY_ORIGIN
))
1531 r2t
= (unsigned long *)(r1t
[i
] & _REGION_ENTRY_ORIGIN
);
1532 __gmap_unshadow_r2t(sg
, raddr
, r2t
);
1533 /* Clear entry and flush translation r1t -> r2t */
1534 gmap_idte_one(asce
, raddr
);
1535 r1t
[i
] = _REGION1_ENTRY_EMPTY
;
1536 /* Free region 2 table */
1537 page
= pfn_to_page(__pa(r2t
) >> PAGE_SHIFT
);
1538 list_del(&page
->lru
);
1539 __free_pages(page
, CRST_ALLOC_ORDER
);
1544 * gmap_unshadow - remove a shadow page table completely
1545 * @sg: pointer to the shadow guest address space structure
1547 * Called with sg->guest_table_lock
1549 static void gmap_unshadow(struct gmap
*sg
)
1551 unsigned long *table
;
1553 BUG_ON(!gmap_is_shadow(sg
));
1557 gmap_call_notifier(sg
, 0, -1UL);
1559 table
= (unsigned long *)(sg
->asce
& _ASCE_ORIGIN
);
1560 switch (sg
->asce
& _ASCE_TYPE_MASK
) {
1561 case _ASCE_TYPE_REGION1
:
1562 __gmap_unshadow_r1t(sg
, 0, table
);
1564 case _ASCE_TYPE_REGION2
:
1565 __gmap_unshadow_r2t(sg
, 0, table
);
1567 case _ASCE_TYPE_REGION3
:
1568 __gmap_unshadow_r3t(sg
, 0, table
);
1570 case _ASCE_TYPE_SEGMENT
:
1571 __gmap_unshadow_sgt(sg
, 0, table
);
1577 * gmap_find_shadow - find a specific asce in the list of shadow tables
1578 * @parent: pointer to the parent gmap
1579 * @asce: ASCE for which the shadow table is created
1580 * @edat_level: edat level to be used for the shadow translation
1582 * Returns the pointer to a gmap if a shadow table with the given asce is
1583 * already available, ERR_PTR(-EAGAIN) if another one is just being created,
1586 static struct gmap
*gmap_find_shadow(struct gmap
*parent
, unsigned long asce
,
1591 list_for_each_entry(sg
, &parent
->children
, list
) {
1592 if (sg
->orig_asce
!= asce
|| sg
->edat_level
!= edat_level
||
1595 if (!sg
->initialized
)
1596 return ERR_PTR(-EAGAIN
);
1597 refcount_inc(&sg
->ref_count
);
1604 * gmap_shadow_valid - check if a shadow guest address space matches the
1605 * given properties and is still valid
1606 * @sg: pointer to the shadow guest address space structure
1607 * @asce: ASCE for which the shadow table is requested
1608 * @edat_level: edat level to be used for the shadow translation
1610 * Returns 1 if the gmap shadow is still valid and matches the given
1611 * properties, the caller can continue using it. Returns 0 otherwise, the
1612 * caller has to request a new shadow gmap in this case.
1615 int gmap_shadow_valid(struct gmap
*sg
, unsigned long asce
, int edat_level
)
1619 return sg
->orig_asce
== asce
&& sg
->edat_level
== edat_level
;
1621 EXPORT_SYMBOL_GPL(gmap_shadow_valid
);
1624 * gmap_shadow - create/find a shadow guest address space
1625 * @parent: pointer to the parent gmap
1626 * @asce: ASCE for which the shadow table is created
1627 * @edat_level: edat level to be used for the shadow translation
1629 * The pages of the top level page table referred by the asce parameter
1630 * will be set to read-only and marked in the PGSTEs of the kvm process.
1631 * The shadow table will be removed automatically on any change to the
1632 * PTE mapping for the source table.
1634 * Returns a guest address space structure, ERR_PTR(-ENOMEM) if out of memory,
1635 * ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the
1636 * parent gmap table could not be protected.
1638 struct gmap
*gmap_shadow(struct gmap
*parent
, unsigned long asce
,
1641 struct gmap
*sg
, *new;
1642 unsigned long limit
;
1645 BUG_ON(parent
->mm
->context
.allow_gmap_hpage_1m
);
1646 BUG_ON(gmap_is_shadow(parent
));
1647 spin_lock(&parent
->shadow_lock
);
1648 sg
= gmap_find_shadow(parent
, asce
, edat_level
);
1649 spin_unlock(&parent
->shadow_lock
);
1652 /* Create a new shadow gmap */
1653 limit
= -1UL >> (33 - (((asce
& _ASCE_TYPE_MASK
) >> 2) * 11));
1654 if (asce
& _ASCE_REAL_SPACE
)
1656 new = gmap_alloc(limit
);
1658 return ERR_PTR(-ENOMEM
);
1659 new->mm
= parent
->mm
;
1660 new->parent
= gmap_get(parent
);
1661 new->orig_asce
= asce
;
1662 new->edat_level
= edat_level
;
1663 new->initialized
= false;
1664 spin_lock(&parent
->shadow_lock
);
1665 /* Recheck if another CPU created the same shadow */
1666 sg
= gmap_find_shadow(parent
, asce
, edat_level
);
1668 spin_unlock(&parent
->shadow_lock
);
1672 if (asce
& _ASCE_REAL_SPACE
) {
1673 /* only allow one real-space gmap shadow */
1674 list_for_each_entry(sg
, &parent
->children
, list
) {
1675 if (sg
->orig_asce
& _ASCE_REAL_SPACE
) {
1676 spin_lock(&sg
->guest_table_lock
);
1678 spin_unlock(&sg
->guest_table_lock
);
1679 list_del(&sg
->list
);
1685 refcount_set(&new->ref_count
, 2);
1686 list_add(&new->list
, &parent
->children
);
1687 if (asce
& _ASCE_REAL_SPACE
) {
1688 /* nothing to protect, return right away */
1689 new->initialized
= true;
1690 spin_unlock(&parent
->shadow_lock
);
1693 spin_unlock(&parent
->shadow_lock
);
1694 /* protect after insertion, so it will get properly invalidated */
1695 down_read(&parent
->mm
->mmap_sem
);
1696 rc
= gmap_protect_range(parent
, asce
& _ASCE_ORIGIN
,
1697 ((asce
& _ASCE_TABLE_LENGTH
) + 1) * PAGE_SIZE
,
1698 PROT_READ
, GMAP_NOTIFY_SHADOW
);
1699 up_read(&parent
->mm
->mmap_sem
);
1700 spin_lock(&parent
->shadow_lock
);
1701 new->initialized
= true;
1703 list_del(&new->list
);
1707 spin_unlock(&parent
->shadow_lock
);
1710 EXPORT_SYMBOL_GPL(gmap_shadow
);
1713 * gmap_shadow_r2t - create an empty shadow region 2 table
1714 * @sg: pointer to the shadow guest address space structure
1715 * @saddr: faulting address in the shadow gmap
1716 * @r2t: parent gmap address of the region 2 table to get shadowed
1717 * @fake: r2t references contiguous guest memory block, not a r2t
1719 * The r2t parameter specifies the address of the source table. The
1720 * four pages of the source table are made read-only in the parent gmap
1721 * address space. A write to the source table area @r2t will automatically
1722 * remove the shadow r2 table and all of its decendents.
1724 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1725 * shadow table structure is incomplete, -ENOMEM if out of memory and
1726 * -EFAULT if an address in the parent gmap could not be resolved.
1728 * Called with sg->mm->mmap_sem in read.
1730 int gmap_shadow_r2t(struct gmap
*sg
, unsigned long saddr
, unsigned long r2t
,
1733 unsigned long raddr
, origin
, offset
, len
;
1734 unsigned long *s_r2t
, *table
;
1738 BUG_ON(!gmap_is_shadow(sg
));
1739 /* Allocate a shadow region second table */
1740 page
= alloc_pages(GFP_KERNEL
, CRST_ALLOC_ORDER
);
1743 page
->index
= r2t
& _REGION_ENTRY_ORIGIN
;
1745 page
->index
|= GMAP_SHADOW_FAKE_TABLE
;
1746 s_r2t
= (unsigned long *) page_to_phys(page
);
1747 /* Install shadow region second table */
1748 spin_lock(&sg
->guest_table_lock
);
1749 table
= gmap_table_walk(sg
, saddr
, 4); /* get region-1 pointer */
1751 rc
= -EAGAIN
; /* Race with unshadow */
1754 if (!(*table
& _REGION_ENTRY_INVALID
)) {
1755 rc
= 0; /* Already established */
1757 } else if (*table
& _REGION_ENTRY_ORIGIN
) {
1758 rc
= -EAGAIN
; /* Race with shadow */
1761 crst_table_init(s_r2t
, _REGION2_ENTRY_EMPTY
);
1762 /* mark as invalid as long as the parent table is not protected */
1763 *table
= (unsigned long) s_r2t
| _REGION_ENTRY_LENGTH
|
1764 _REGION_ENTRY_TYPE_R1
| _REGION_ENTRY_INVALID
;
1765 if (sg
->edat_level
>= 1)
1766 *table
|= (r2t
& _REGION_ENTRY_PROTECT
);
1767 list_add(&page
->lru
, &sg
->crst_list
);
1769 /* nothing to protect for fake tables */
1770 *table
&= ~_REGION_ENTRY_INVALID
;
1771 spin_unlock(&sg
->guest_table_lock
);
1774 spin_unlock(&sg
->guest_table_lock
);
1775 /* Make r2t read-only in parent gmap page table */
1776 raddr
= (saddr
& _REGION1_MASK
) | _SHADOW_RMAP_REGION1
;
1777 origin
= r2t
& _REGION_ENTRY_ORIGIN
;
1778 offset
= ((r2t
& _REGION_ENTRY_OFFSET
) >> 6) * PAGE_SIZE
;
1779 len
= ((r2t
& _REGION_ENTRY_LENGTH
) + 1) * PAGE_SIZE
- offset
;
1780 rc
= gmap_protect_rmap(sg
, raddr
, origin
+ offset
, len
);
1781 spin_lock(&sg
->guest_table_lock
);
1783 table
= gmap_table_walk(sg
, saddr
, 4);
1784 if (!table
|| (*table
& _REGION_ENTRY_ORIGIN
) !=
1785 (unsigned long) s_r2t
)
1786 rc
= -EAGAIN
; /* Race with unshadow */
1788 *table
&= ~_REGION_ENTRY_INVALID
;
1790 gmap_unshadow_r2t(sg
, raddr
);
1792 spin_unlock(&sg
->guest_table_lock
);
1795 spin_unlock(&sg
->guest_table_lock
);
1796 __free_pages(page
, CRST_ALLOC_ORDER
);
1799 EXPORT_SYMBOL_GPL(gmap_shadow_r2t
);
1802 * gmap_shadow_r3t - create a shadow region 3 table
1803 * @sg: pointer to the shadow guest address space structure
1804 * @saddr: faulting address in the shadow gmap
1805 * @r3t: parent gmap address of the region 3 table to get shadowed
1806 * @fake: r3t references contiguous guest memory block, not a r3t
1808 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
1809 * shadow table structure is incomplete, -ENOMEM if out of memory and
1810 * -EFAULT if an address in the parent gmap could not be resolved.
1812 * Called with sg->mm->mmap_sem in read.
1814 int gmap_shadow_r3t(struct gmap
*sg
, unsigned long saddr
, unsigned long r3t
,
1817 unsigned long raddr
, origin
, offset
, len
;
1818 unsigned long *s_r3t
, *table
;
1822 BUG_ON(!gmap_is_shadow(sg
));
1823 /* Allocate a shadow region second table */
1824 page
= alloc_pages(GFP_KERNEL
, CRST_ALLOC_ORDER
);
1827 page
->index
= r3t
& _REGION_ENTRY_ORIGIN
;
1829 page
->index
|= GMAP_SHADOW_FAKE_TABLE
;
1830 s_r3t
= (unsigned long *) page_to_phys(page
);
1831 /* Install shadow region second table */
1832 spin_lock(&sg
->guest_table_lock
);
1833 table
= gmap_table_walk(sg
, saddr
, 3); /* get region-2 pointer */
1835 rc
= -EAGAIN
; /* Race with unshadow */
1838 if (!(*table
& _REGION_ENTRY_INVALID
)) {
1839 rc
= 0; /* Already established */
1841 } else if (*table
& _REGION_ENTRY_ORIGIN
) {
1842 rc
= -EAGAIN
; /* Race with shadow */
1844 crst_table_init(s_r3t
, _REGION3_ENTRY_EMPTY
);
1845 /* mark as invalid as long as the parent table is not protected */
1846 *table
= (unsigned long) s_r3t
| _REGION_ENTRY_LENGTH
|
1847 _REGION_ENTRY_TYPE_R2
| _REGION_ENTRY_INVALID
;
1848 if (sg
->edat_level
>= 1)
1849 *table
|= (r3t
& _REGION_ENTRY_PROTECT
);
1850 list_add(&page
->lru
, &sg
->crst_list
);
1852 /* nothing to protect for fake tables */
1853 *table
&= ~_REGION_ENTRY_INVALID
;
1854 spin_unlock(&sg
->guest_table_lock
);
1857 spin_unlock(&sg
->guest_table_lock
);
1858 /* Make r3t read-only in parent gmap page table */
1859 raddr
= (saddr
& _REGION2_MASK
) | _SHADOW_RMAP_REGION2
;
1860 origin
= r3t
& _REGION_ENTRY_ORIGIN
;
1861 offset
= ((r3t
& _REGION_ENTRY_OFFSET
) >> 6) * PAGE_SIZE
;
1862 len
= ((r3t
& _REGION_ENTRY_LENGTH
) + 1) * PAGE_SIZE
- offset
;
1863 rc
= gmap_protect_rmap(sg
, raddr
, origin
+ offset
, len
);
1864 spin_lock(&sg
->guest_table_lock
);
1866 table
= gmap_table_walk(sg
, saddr
, 3);
1867 if (!table
|| (*table
& _REGION_ENTRY_ORIGIN
) !=
1868 (unsigned long) s_r3t
)
1869 rc
= -EAGAIN
; /* Race with unshadow */
1871 *table
&= ~_REGION_ENTRY_INVALID
;
1873 gmap_unshadow_r3t(sg
, raddr
);
1875 spin_unlock(&sg
->guest_table_lock
);
1878 spin_unlock(&sg
->guest_table_lock
);
1879 __free_pages(page
, CRST_ALLOC_ORDER
);
1882 EXPORT_SYMBOL_GPL(gmap_shadow_r3t
);
1885 * gmap_shadow_sgt - create a shadow segment table
1886 * @sg: pointer to the shadow guest address space structure
1887 * @saddr: faulting address in the shadow gmap
1888 * @sgt: parent gmap address of the segment table to get shadowed
1889 * @fake: sgt references contiguous guest memory block, not a sgt
1891 * Returns: 0 if successfully shadowed or already shadowed, -EAGAIN if the
1892 * shadow table structure is incomplete, -ENOMEM if out of memory and
1893 * -EFAULT if an address in the parent gmap could not be resolved.
1895 * Called with sg->mm->mmap_sem in read.
1897 int gmap_shadow_sgt(struct gmap
*sg
, unsigned long saddr
, unsigned long sgt
,
1900 unsigned long raddr
, origin
, offset
, len
;
1901 unsigned long *s_sgt
, *table
;
1905 BUG_ON(!gmap_is_shadow(sg
) || (sgt
& _REGION3_ENTRY_LARGE
));
1906 /* Allocate a shadow segment table */
1907 page
= alloc_pages(GFP_KERNEL
, CRST_ALLOC_ORDER
);
1910 page
->index
= sgt
& _REGION_ENTRY_ORIGIN
;
1912 page
->index
|= GMAP_SHADOW_FAKE_TABLE
;
1913 s_sgt
= (unsigned long *) page_to_phys(page
);
1914 /* Install shadow region second table */
1915 spin_lock(&sg
->guest_table_lock
);
1916 table
= gmap_table_walk(sg
, saddr
, 2); /* get region-3 pointer */
1918 rc
= -EAGAIN
; /* Race with unshadow */
1921 if (!(*table
& _REGION_ENTRY_INVALID
)) {
1922 rc
= 0; /* Already established */
1924 } else if (*table
& _REGION_ENTRY_ORIGIN
) {
1925 rc
= -EAGAIN
; /* Race with shadow */
1928 crst_table_init(s_sgt
, _SEGMENT_ENTRY_EMPTY
);
1929 /* mark as invalid as long as the parent table is not protected */
1930 *table
= (unsigned long) s_sgt
| _REGION_ENTRY_LENGTH
|
1931 _REGION_ENTRY_TYPE_R3
| _REGION_ENTRY_INVALID
;
1932 if (sg
->edat_level
>= 1)
1933 *table
|= sgt
& _REGION_ENTRY_PROTECT
;
1934 list_add(&page
->lru
, &sg
->crst_list
);
1936 /* nothing to protect for fake tables */
1937 *table
&= ~_REGION_ENTRY_INVALID
;
1938 spin_unlock(&sg
->guest_table_lock
);
1941 spin_unlock(&sg
->guest_table_lock
);
1942 /* Make sgt read-only in parent gmap page table */
1943 raddr
= (saddr
& _REGION3_MASK
) | _SHADOW_RMAP_REGION3
;
1944 origin
= sgt
& _REGION_ENTRY_ORIGIN
;
1945 offset
= ((sgt
& _REGION_ENTRY_OFFSET
) >> 6) * PAGE_SIZE
;
1946 len
= ((sgt
& _REGION_ENTRY_LENGTH
) + 1) * PAGE_SIZE
- offset
;
1947 rc
= gmap_protect_rmap(sg
, raddr
, origin
+ offset
, len
);
1948 spin_lock(&sg
->guest_table_lock
);
1950 table
= gmap_table_walk(sg
, saddr
, 2);
1951 if (!table
|| (*table
& _REGION_ENTRY_ORIGIN
) !=
1952 (unsigned long) s_sgt
)
1953 rc
= -EAGAIN
; /* Race with unshadow */
1955 *table
&= ~_REGION_ENTRY_INVALID
;
1957 gmap_unshadow_sgt(sg
, raddr
);
1959 spin_unlock(&sg
->guest_table_lock
);
1962 spin_unlock(&sg
->guest_table_lock
);
1963 __free_pages(page
, CRST_ALLOC_ORDER
);
1966 EXPORT_SYMBOL_GPL(gmap_shadow_sgt
);
1969 * gmap_shadow_lookup_pgtable - find a shadow page table
1970 * @sg: pointer to the shadow guest address space structure
1971 * @saddr: the address in the shadow aguest address space
1972 * @pgt: parent gmap address of the page table to get shadowed
1973 * @dat_protection: if the pgtable is marked as protected by dat
1974 * @fake: pgt references contiguous guest memory block, not a pgtable
1976 * Returns 0 if the shadow page table was found and -EAGAIN if the page
1977 * table was not found.
1979 * Called with sg->mm->mmap_sem in read.
1981 int gmap_shadow_pgt_lookup(struct gmap
*sg
, unsigned long saddr
,
1982 unsigned long *pgt
, int *dat_protection
,
1985 unsigned long *table
;
1989 BUG_ON(!gmap_is_shadow(sg
));
1990 spin_lock(&sg
->guest_table_lock
);
1991 table
= gmap_table_walk(sg
, saddr
, 1); /* get segment pointer */
1992 if (table
&& !(*table
& _SEGMENT_ENTRY_INVALID
)) {
1993 /* Shadow page tables are full pages (pte+pgste) */
1994 page
= pfn_to_page(*table
>> PAGE_SHIFT
);
1995 *pgt
= page
->index
& ~GMAP_SHADOW_FAKE_TABLE
;
1996 *dat_protection
= !!(*table
& _SEGMENT_ENTRY_PROTECT
);
1997 *fake
= !!(page
->index
& GMAP_SHADOW_FAKE_TABLE
);
2002 spin_unlock(&sg
->guest_table_lock
);
2006 EXPORT_SYMBOL_GPL(gmap_shadow_pgt_lookup
);
2009 * gmap_shadow_pgt - instantiate a shadow page table
2010 * @sg: pointer to the shadow guest address space structure
2011 * @saddr: faulting address in the shadow gmap
2012 * @pgt: parent gmap address of the page table to get shadowed
2013 * @fake: pgt references contiguous guest memory block, not a pgtable
2015 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
2016 * shadow table structure is incomplete, -ENOMEM if out of memory,
2017 * -EFAULT if an address in the parent gmap could not be resolved and
2019 * Called with gmap->mm->mmap_sem in read
2021 int gmap_shadow_pgt(struct gmap
*sg
, unsigned long saddr
, unsigned long pgt
,
2024 unsigned long raddr
, origin
;
2025 unsigned long *s_pgt
, *table
;
2029 BUG_ON(!gmap_is_shadow(sg
) || (pgt
& _SEGMENT_ENTRY_LARGE
));
2030 /* Allocate a shadow page table */
2031 page
= page_table_alloc_pgste(sg
->mm
);
2034 page
->index
= pgt
& _SEGMENT_ENTRY_ORIGIN
;
2036 page
->index
|= GMAP_SHADOW_FAKE_TABLE
;
2037 s_pgt
= (unsigned long *) page_to_phys(page
);
2038 /* Install shadow page table */
2039 spin_lock(&sg
->guest_table_lock
);
2040 table
= gmap_table_walk(sg
, saddr
, 1); /* get segment pointer */
2042 rc
= -EAGAIN
; /* Race with unshadow */
2045 if (!(*table
& _SEGMENT_ENTRY_INVALID
)) {
2046 rc
= 0; /* Already established */
2048 } else if (*table
& _SEGMENT_ENTRY_ORIGIN
) {
2049 rc
= -EAGAIN
; /* Race with shadow */
2052 /* mark as invalid as long as the parent table is not protected */
2053 *table
= (unsigned long) s_pgt
| _SEGMENT_ENTRY
|
2054 (pgt
& _SEGMENT_ENTRY_PROTECT
) | _SEGMENT_ENTRY_INVALID
;
2055 list_add(&page
->lru
, &sg
->pt_list
);
2057 /* nothing to protect for fake tables */
2058 *table
&= ~_SEGMENT_ENTRY_INVALID
;
2059 spin_unlock(&sg
->guest_table_lock
);
2062 spin_unlock(&sg
->guest_table_lock
);
2063 /* Make pgt read-only in parent gmap page table (not the pgste) */
2064 raddr
= (saddr
& _SEGMENT_MASK
) | _SHADOW_RMAP_SEGMENT
;
2065 origin
= pgt
& _SEGMENT_ENTRY_ORIGIN
& PAGE_MASK
;
2066 rc
= gmap_protect_rmap(sg
, raddr
, origin
, PAGE_SIZE
);
2067 spin_lock(&sg
->guest_table_lock
);
2069 table
= gmap_table_walk(sg
, saddr
, 1);
2070 if (!table
|| (*table
& _SEGMENT_ENTRY_ORIGIN
) !=
2071 (unsigned long) s_pgt
)
2072 rc
= -EAGAIN
; /* Race with unshadow */
2074 *table
&= ~_SEGMENT_ENTRY_INVALID
;
2076 gmap_unshadow_pgt(sg
, raddr
);
2078 spin_unlock(&sg
->guest_table_lock
);
2081 spin_unlock(&sg
->guest_table_lock
);
2082 page_table_free_pgste(page
);
2086 EXPORT_SYMBOL_GPL(gmap_shadow_pgt
);
2089 * gmap_shadow_page - create a shadow page mapping
2090 * @sg: pointer to the shadow guest address space structure
2091 * @saddr: faulting address in the shadow gmap
2092 * @pte: pte in parent gmap address space to get shadowed
2094 * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
2095 * shadow table structure is incomplete, -ENOMEM if out of memory and
2096 * -EFAULT if an address in the parent gmap could not be resolved.
2098 * Called with sg->mm->mmap_sem in read.
2100 int gmap_shadow_page(struct gmap
*sg
, unsigned long saddr
, pte_t pte
)
2102 struct gmap
*parent
;
2103 struct gmap_rmap
*rmap
;
2104 unsigned long vmaddr
, paddr
;
2106 pte_t
*sptep
, *tptep
;
2110 BUG_ON(!gmap_is_shadow(sg
));
2111 parent
= sg
->parent
;
2112 prot
= (pte_val(pte
) & _PAGE_PROTECT
) ? PROT_READ
: PROT_WRITE
;
2114 rmap
= kzalloc(sizeof(*rmap
), GFP_KERNEL
);
2117 rmap
->raddr
= (saddr
& PAGE_MASK
) | _SHADOW_RMAP_PGTABLE
;
2120 paddr
= pte_val(pte
) & PAGE_MASK
;
2121 vmaddr
= __gmap_translate(parent
, paddr
);
2122 if (IS_ERR_VALUE(vmaddr
)) {
2126 rc
= radix_tree_preload(GFP_KERNEL
);
2130 sptep
= gmap_pte_op_walk(parent
, paddr
, &ptl
);
2132 spin_lock(&sg
->guest_table_lock
);
2133 /* Get page table pointer */
2134 tptep
= (pte_t
*) gmap_table_walk(sg
, saddr
, 0);
2136 spin_unlock(&sg
->guest_table_lock
);
2137 gmap_pte_op_end(ptl
);
2138 radix_tree_preload_end();
2141 rc
= ptep_shadow_pte(sg
->mm
, saddr
, sptep
, tptep
, pte
);
2143 /* Success and a new mapping */
2144 gmap_insert_rmap(sg
, vmaddr
, rmap
);
2148 gmap_pte_op_end(ptl
);
2149 spin_unlock(&sg
->guest_table_lock
);
2151 radix_tree_preload_end();
2154 rc
= gmap_pte_op_fixup(parent
, paddr
, vmaddr
, prot
);
2161 EXPORT_SYMBOL_GPL(gmap_shadow_page
);
2164 * gmap_shadow_notify - handle notifications for shadow gmap
2166 * Called with sg->parent->shadow_lock.
2168 static void gmap_shadow_notify(struct gmap
*sg
, unsigned long vmaddr
,
2169 unsigned long gaddr
)
2171 struct gmap_rmap
*rmap
, *rnext
, *head
;
2172 unsigned long start
, end
, bits
, raddr
;
2174 BUG_ON(!gmap_is_shadow(sg
));
2176 spin_lock(&sg
->guest_table_lock
);
2178 spin_unlock(&sg
->guest_table_lock
);
2181 /* Check for top level table */
2182 start
= sg
->orig_asce
& _ASCE_ORIGIN
;
2183 end
= start
+ ((sg
->orig_asce
& _ASCE_TABLE_LENGTH
) + 1) * PAGE_SIZE
;
2184 if (!(sg
->orig_asce
& _ASCE_REAL_SPACE
) && gaddr
>= start
&&
2186 /* The complete shadow table has to go */
2188 spin_unlock(&sg
->guest_table_lock
);
2189 list_del(&sg
->list
);
2193 /* Remove the page table tree from on specific entry */
2194 head
= radix_tree_delete(&sg
->host_to_rmap
, vmaddr
>> PAGE_SHIFT
);
2195 gmap_for_each_rmap_safe(rmap
, rnext
, head
) {
2196 bits
= rmap
->raddr
& _SHADOW_RMAP_MASK
;
2197 raddr
= rmap
->raddr
^ bits
;
2199 case _SHADOW_RMAP_REGION1
:
2200 gmap_unshadow_r2t(sg
, raddr
);
2202 case _SHADOW_RMAP_REGION2
:
2203 gmap_unshadow_r3t(sg
, raddr
);
2205 case _SHADOW_RMAP_REGION3
:
2206 gmap_unshadow_sgt(sg
, raddr
);
2208 case _SHADOW_RMAP_SEGMENT
:
2209 gmap_unshadow_pgt(sg
, raddr
);
2211 case _SHADOW_RMAP_PGTABLE
:
2212 gmap_unshadow_page(sg
, raddr
);
2217 spin_unlock(&sg
->guest_table_lock
);
2221 * ptep_notify - call all invalidation callbacks for a specific pte.
2222 * @mm: pointer to the process mm_struct
2223 * @addr: virtual address in the process address space
2224 * @pte: pointer to the page table entry
2225 * @bits: bits from the pgste that caused the notify call
2227 * This function is assumed to be called with the page table lock held
2228 * for the pte to notify.
2230 void ptep_notify(struct mm_struct
*mm
, unsigned long vmaddr
,
2231 pte_t
*pte
, unsigned long bits
)
2233 unsigned long offset
, gaddr
= 0;
2234 unsigned long *table
;
2235 struct gmap
*gmap
, *sg
, *next
;
2237 offset
= ((unsigned long) pte
) & (255 * sizeof(pte_t
));
2238 offset
= offset
* (PAGE_SIZE
/ sizeof(pte_t
));
2240 list_for_each_entry_rcu(gmap
, &mm
->context
.gmap_list
, list
) {
2241 spin_lock(&gmap
->guest_table_lock
);
2242 table
= radix_tree_lookup(&gmap
->host_to_guest
,
2243 vmaddr
>> PMD_SHIFT
);
2245 gaddr
= __gmap_segment_gaddr(table
) + offset
;
2246 spin_unlock(&gmap
->guest_table_lock
);
2250 if (!list_empty(&gmap
->children
) && (bits
& PGSTE_VSIE_BIT
)) {
2251 spin_lock(&gmap
->shadow_lock
);
2252 list_for_each_entry_safe(sg
, next
,
2253 &gmap
->children
, list
)
2254 gmap_shadow_notify(sg
, vmaddr
, gaddr
);
2255 spin_unlock(&gmap
->shadow_lock
);
2257 if (bits
& PGSTE_IN_BIT
)
2258 gmap_call_notifier(gmap
, gaddr
, gaddr
+ PAGE_SIZE
- 1);
2262 EXPORT_SYMBOL_GPL(ptep_notify
);
2264 static void pmdp_notify_gmap(struct gmap
*gmap
, pmd_t
*pmdp
,
2265 unsigned long gaddr
)
2267 pmd_val(*pmdp
) &= ~_SEGMENT_ENTRY_GMAP_IN
;
2268 gmap_call_notifier(gmap
, gaddr
, gaddr
+ HPAGE_SIZE
- 1);
2272 * gmap_pmdp_xchg - exchange a gmap pmd with another
2273 * @gmap: pointer to the guest address space structure
2274 * @pmdp: pointer to the pmd entry
2275 * @new: replacement entry
2276 * @gaddr: the affected guest address
2278 * This function is assumed to be called with the guest_table_lock
2281 static void gmap_pmdp_xchg(struct gmap
*gmap
, pmd_t
*pmdp
, pmd_t
new,
2282 unsigned long gaddr
)
2284 gaddr
&= HPAGE_MASK
;
2285 pmdp_notify_gmap(gmap
, pmdp
, gaddr
);
2286 pmd_val(new) &= ~_SEGMENT_ENTRY_GMAP_IN
;
2287 if (MACHINE_HAS_TLB_GUEST
)
2288 __pmdp_idte(gaddr
, (pmd_t
*)pmdp
, IDTE_GUEST_ASCE
, gmap
->asce
,
2290 else if (MACHINE_HAS_IDTE
)
2291 __pmdp_idte(gaddr
, (pmd_t
*)pmdp
, 0, 0, IDTE_GLOBAL
);
2297 static void gmap_pmdp_clear(struct mm_struct
*mm
, unsigned long vmaddr
,
2302 unsigned long gaddr
;
2305 list_for_each_entry_rcu(gmap
, &mm
->context
.gmap_list
, list
) {
2306 spin_lock(&gmap
->guest_table_lock
);
2307 pmdp
= (pmd_t
*)radix_tree_delete(&gmap
->host_to_guest
,
2308 vmaddr
>> PMD_SHIFT
);
2310 gaddr
= __gmap_segment_gaddr((unsigned long *)pmdp
);
2311 pmdp_notify_gmap(gmap
, pmdp
, gaddr
);
2312 WARN_ON(pmd_val(*pmdp
) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE
|
2313 _SEGMENT_ENTRY_GMAP_UC
));
2316 pmd_val(*pmdp
) = _SEGMENT_ENTRY_EMPTY
;
2318 spin_unlock(&gmap
->guest_table_lock
);
2324 * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
2326 * @mm: pointer to the process mm_struct
2327 * @vmaddr: virtual address in the process address space
2329 void gmap_pmdp_invalidate(struct mm_struct
*mm
, unsigned long vmaddr
)
2331 gmap_pmdp_clear(mm
, vmaddr
, 0);
2333 EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate
);
2336 * gmap_pmdp_csp - csp all affected guest pmd entries
2337 * @mm: pointer to the process mm_struct
2338 * @vmaddr: virtual address in the process address space
2340 void gmap_pmdp_csp(struct mm_struct
*mm
, unsigned long vmaddr
)
2342 gmap_pmdp_clear(mm
, vmaddr
, 1);
2344 EXPORT_SYMBOL_GPL(gmap_pmdp_csp
);
2347 * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
2348 * @mm: pointer to the process mm_struct
2349 * @vmaddr: virtual address in the process address space
2351 void gmap_pmdp_idte_local(struct mm_struct
*mm
, unsigned long vmaddr
)
2353 unsigned long *entry
, gaddr
;
2358 list_for_each_entry_rcu(gmap
, &mm
->context
.gmap_list
, list
) {
2359 spin_lock(&gmap
->guest_table_lock
);
2360 entry
= radix_tree_delete(&gmap
->host_to_guest
,
2361 vmaddr
>> PMD_SHIFT
);
2363 pmdp
= (pmd_t
*)entry
;
2364 gaddr
= __gmap_segment_gaddr(entry
);
2365 pmdp_notify_gmap(gmap
, pmdp
, gaddr
);
2366 WARN_ON(*entry
& ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE
|
2367 _SEGMENT_ENTRY_GMAP_UC
));
2368 if (MACHINE_HAS_TLB_GUEST
)
2369 __pmdp_idte(gaddr
, pmdp
, IDTE_GUEST_ASCE
,
2370 gmap
->asce
, IDTE_LOCAL
);
2371 else if (MACHINE_HAS_IDTE
)
2372 __pmdp_idte(gaddr
, pmdp
, 0, 0, IDTE_LOCAL
);
2373 *entry
= _SEGMENT_ENTRY_EMPTY
;
2375 spin_unlock(&gmap
->guest_table_lock
);
2379 EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local
);
2382 * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
2383 * @mm: pointer to the process mm_struct
2384 * @vmaddr: virtual address in the process address space
2386 void gmap_pmdp_idte_global(struct mm_struct
*mm
, unsigned long vmaddr
)
2388 unsigned long *entry
, gaddr
;
2393 list_for_each_entry_rcu(gmap
, &mm
->context
.gmap_list
, list
) {
2394 spin_lock(&gmap
->guest_table_lock
);
2395 entry
= radix_tree_delete(&gmap
->host_to_guest
,
2396 vmaddr
>> PMD_SHIFT
);
2398 pmdp
= (pmd_t
*)entry
;
2399 gaddr
= __gmap_segment_gaddr(entry
);
2400 pmdp_notify_gmap(gmap
, pmdp
, gaddr
);
2401 WARN_ON(*entry
& ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE
|
2402 _SEGMENT_ENTRY_GMAP_UC
));
2403 if (MACHINE_HAS_TLB_GUEST
)
2404 __pmdp_idte(gaddr
, pmdp
, IDTE_GUEST_ASCE
,
2405 gmap
->asce
, IDTE_GLOBAL
);
2406 else if (MACHINE_HAS_IDTE
)
2407 __pmdp_idte(gaddr
, pmdp
, 0, 0, IDTE_GLOBAL
);
2410 *entry
= _SEGMENT_ENTRY_EMPTY
;
2412 spin_unlock(&gmap
->guest_table_lock
);
2416 EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global
);
2419 * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status
2420 * @gmap: pointer to guest address space
2421 * @pmdp: pointer to the pmd to be tested
2422 * @gaddr: virtual address in the guest address space
2424 * This function is assumed to be called with the guest_table_lock
2427 static bool gmap_test_and_clear_dirty_pmd(struct gmap
*gmap
, pmd_t
*pmdp
,
2428 unsigned long gaddr
)
2430 if (pmd_val(*pmdp
) & _SEGMENT_ENTRY_INVALID
)
2433 /* Already protected memory, which did not change is clean */
2434 if (pmd_val(*pmdp
) & _SEGMENT_ENTRY_PROTECT
&&
2435 !(pmd_val(*pmdp
) & _SEGMENT_ENTRY_GMAP_UC
))
2438 /* Clear UC indication and reset protection */
2439 pmd_val(*pmdp
) &= ~_SEGMENT_ENTRY_GMAP_UC
;
2440 gmap_protect_pmd(gmap
, gaddr
, pmdp
, PROT_READ
, 0);
2445 * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment
2446 * @gmap: pointer to guest address space
2447 * @bitmap: dirty bitmap for this pmd
2448 * @gaddr: virtual address in the guest address space
2449 * @vmaddr: virtual address in the host address space
2451 * This function is assumed to be called with the guest_table_lock
2454 void gmap_sync_dirty_log_pmd(struct gmap
*gmap
, unsigned long bitmap
[4],
2455 unsigned long gaddr
, unsigned long vmaddr
)
2462 pmdp
= gmap_pmd_op_walk(gmap
, gaddr
);
2466 if (pmd_large(*pmdp
)) {
2467 if (gmap_test_and_clear_dirty_pmd(gmap
, pmdp
, gaddr
))
2468 bitmap_fill(bitmap
, _PAGE_ENTRIES
);
2470 for (i
= 0; i
< _PAGE_ENTRIES
; i
++, vmaddr
+= PAGE_SIZE
) {
2471 ptep
= pte_alloc_map_lock(gmap
->mm
, pmdp
, vmaddr
, &ptl
);
2474 if (ptep_test_and_clear_uc(gmap
->mm
, vmaddr
, ptep
))
2479 gmap_pmd_op_end(gmap
, pmdp
);
2481 EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd
);
2483 static inline void thp_split_mm(struct mm_struct
*mm
)
2485 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2486 struct vm_area_struct
*vma
;
2489 for (vma
= mm
->mmap
; vma
!= NULL
; vma
= vma
->vm_next
) {
2490 for (addr
= vma
->vm_start
;
2493 follow_page(vma
, addr
, FOLL_SPLIT
);
2494 vma
->vm_flags
&= ~VM_HUGEPAGE
;
2495 vma
->vm_flags
|= VM_NOHUGEPAGE
;
2497 mm
->def_flags
|= VM_NOHUGEPAGE
;
2502 * Remove all empty zero pages from the mapping for lazy refaulting
2503 * - This must be called after mm->context.has_pgste is set, to avoid
2504 * future creation of zero pages
2505 * - This must be called after THP was enabled
2507 static int __zap_zero_pages(pmd_t
*pmd
, unsigned long start
,
2508 unsigned long end
, struct mm_walk
*walk
)
2512 for (addr
= start
; addr
!= end
; addr
+= PAGE_SIZE
) {
2516 ptep
= pte_offset_map_lock(walk
->mm
, pmd
, addr
, &ptl
);
2517 if (is_zero_pfn(pte_pfn(*ptep
)))
2518 ptep_xchg_direct(walk
->mm
, addr
, ptep
, __pte(_PAGE_INVALID
));
2519 pte_unmap_unlock(ptep
, ptl
);
2524 static const struct mm_walk_ops zap_zero_walk_ops
= {
2525 .pmd_entry
= __zap_zero_pages
,
2529 * switch on pgstes for its userspace process (for kvm)
2531 int s390_enable_sie(void)
2533 struct mm_struct
*mm
= current
->mm
;
2535 /* Do we have pgstes? if yes, we are done */
2536 if (mm_has_pgste(mm
))
2538 /* Fail if the page tables are 2K */
2539 if (!mm_alloc_pgste(mm
))
2541 down_write(&mm
->mmap_sem
);
2542 mm
->context
.has_pgste
= 1;
2543 /* split thp mappings and disable thp for future mappings */
2545 walk_page_range(mm
, 0, TASK_SIZE
, &zap_zero_walk_ops
, NULL
);
2546 up_write(&mm
->mmap_sem
);
2549 EXPORT_SYMBOL_GPL(s390_enable_sie
);
2552 * Enable storage key handling from now on and initialize the storage
2553 * keys with the default key.
2555 static int __s390_enable_skey_pte(pte_t
*pte
, unsigned long addr
,
2556 unsigned long next
, struct mm_walk
*walk
)
2558 /* Clear storage key */
2559 ptep_zap_key(walk
->mm
, addr
, pte
);
2563 static int __s390_enable_skey_hugetlb(pte_t
*pte
, unsigned long addr
,
2564 unsigned long hmask
, unsigned long next
,
2565 struct mm_walk
*walk
)
2567 pmd_t
*pmd
= (pmd_t
*)pte
;
2568 unsigned long start
, end
;
2569 struct page
*page
= pmd_page(*pmd
);
2572 * The write check makes sure we do not set a key on shared
2573 * memory. This is needed as the walker does not differentiate
2574 * between actual guest memory and the process executable or
2577 if (pmd_val(*pmd
) & _SEGMENT_ENTRY_INVALID
||
2578 !(pmd_val(*pmd
) & _SEGMENT_ENTRY_WRITE
))
2581 start
= pmd_val(*pmd
) & HPAGE_MASK
;
2582 end
= start
+ HPAGE_SIZE
- 1;
2583 __storage_key_init_range(start
, end
);
2584 set_bit(PG_arch_1
, &page
->flags
);
2588 static const struct mm_walk_ops enable_skey_walk_ops
= {
2589 .hugetlb_entry
= __s390_enable_skey_hugetlb
,
2590 .pte_entry
= __s390_enable_skey_pte
,
2593 int s390_enable_skey(void)
2595 struct mm_struct
*mm
= current
->mm
;
2596 struct vm_area_struct
*vma
;
2599 down_write(&mm
->mmap_sem
);
2600 if (mm_uses_skeys(mm
))
2603 mm
->context
.uses_skeys
= 1;
2604 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
) {
2605 if (ksm_madvise(vma
, vma
->vm_start
, vma
->vm_end
,
2606 MADV_UNMERGEABLE
, &vma
->vm_flags
)) {
2607 mm
->context
.uses_skeys
= 0;
2612 mm
->def_flags
&= ~VM_MERGEABLE
;
2614 walk_page_range(mm
, 0, TASK_SIZE
, &enable_skey_walk_ops
, NULL
);
2617 up_write(&mm
->mmap_sem
);
2620 EXPORT_SYMBOL_GPL(s390_enable_skey
);
2623 * Reset CMMA state, make all pages stable again.
2625 static int __s390_reset_cmma(pte_t
*pte
, unsigned long addr
,
2626 unsigned long next
, struct mm_walk
*walk
)
2628 ptep_zap_unused(walk
->mm
, addr
, pte
, 1);
2632 static const struct mm_walk_ops reset_cmma_walk_ops
= {
2633 .pte_entry
= __s390_reset_cmma
,
2636 void s390_reset_cmma(struct mm_struct
*mm
)
2638 down_write(&mm
->mmap_sem
);
2639 walk_page_range(mm
, 0, TASK_SIZE
, &reset_cmma_walk_ops
, NULL
);
2640 up_write(&mm
->mmap_sem
);
2642 EXPORT_SYMBOL_GPL(s390_reset_cmma
);