arch/arm64/mm/copypage.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Based on arch/arm/mm/copypage.c
   4  *
   5  * Copyright (C) 2002 Deep Blue Solutions Ltd, All Rights Reserved.
   6  * Copyright (C) 2012 ARM Ltd.
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/mm.h>
  11
  12 #include <asm/page.h>
  13 #include <asm/cacheflush.h>
  14 #include <asm/cpufeature.h>
  15 #include <asm/mte.h>
  16
  17 void copy_highpage(struct page *to, struct page *from)
  18 {
  19         void *kto = page_address(to);
  20         void *kfrom = page_address(from);
  21         struct folio *src = page_folio(from);
  22         struct folio *dst = page_folio(to);
  23         unsigned int i, nr_pages;
  24
  25         copy_page(kto, kfrom);
  26
  27         if (kasan_hw_tags_enabled())
  28                 page_kasan_tag_reset(to);
  29
  30         if (!system_supports_mte())
  31                 return;
  32
  33         if (folio_test_hugetlb(src) &&
  34             folio_test_hugetlb_mte_tagged(src)) {
  35                 if (!folio_try_hugetlb_mte_tagging(dst))
  36                         return;
  37
  38                 /*
  39                  * Populate tags for all subpages.
  40                  *
  41                  * Don't assume the first page is head page since
  42                  * huge page copy may start from any subpage.
  43                  */
  44                 nr_pages = folio_nr_pages(src);
  45                 for (i = 0; i < nr_pages; i++) {
  46                         kfrom = page_address(folio_page(src, i));
  47                         kto = page_address(folio_page(dst, i));
  48                         mte_copy_page_tags(kto, kfrom);
  49                 }
  50                 folio_set_hugetlb_mte_tagged(dst);
  51         } else if (page_mte_tagged(from)) {
  52                 /* It's a new page, shouldn't have been tagged yet */
  53                 WARN_ON_ONCE(!try_page_mte_tagging(to));
  54
  55                 mte_copy_page_tags(kto, kfrom);
  56                 set_page_mte_tagged(to);
  57         }
  58 }
  59 EXPORT_SYMBOL(copy_highpage);
  60
  61 void copy_user_highpage(struct page *to, struct page *from,
  62                         unsigned long vaddr, struct vm_area_struct *vma)
  63 {
  64         copy_highpage(to, from);
  65         flush_dcache_page(to);
  66 }
  67 EXPORT_SYMBOL_GPL(copy_user_highpage);