arch/arm/mm/flush.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/mm/flush.c
   4  *
   5  *  Copyright (C) 1995-2002 Russell King
   6  */
   7 #include <linux/module.h>
   8 #include <linux/mm.h>
   9 #include <linux/pagemap.h>
  10 #include <linux/highmem.h>
  11
  12 #include <asm/cacheflush.h>
  13 #include <asm/cachetype.h>
  14 #include <asm/highmem.h>
  15 #include <asm/smp_plat.h>
  16 #include <asm/tlbflush.h>
  17 #include <linux/hugetlb.h>
  18
  19 #include "mm.h"
  20
  21 #ifdef CONFIG_ARM_HEAVY_MB
  22 void (*soc_mb)(void);
  23
  24 void arm_heavy_mb(void)
  25 {
  26 #ifdef CONFIG_OUTER_CACHE_SYNC
  27         if (outer_cache.sync)
  28                 outer_cache.sync();
  29 #endif
  30         if (soc_mb)
  31                 soc_mb();
  32 }
  33 EXPORT_SYMBOL(arm_heavy_mb);
  34 #endif
  35
  36 #ifdef CONFIG_CPU_CACHE_VIPT
  37
  38 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  39 {
  40         unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  41         const int zero = 0;
  42
  43         set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
  44
  45         asm(    "mcrr   p15, 0, %1, %0, c14\n"
  46         "       mcr     p15, 0, %2, c7, c10, 4"
  47             :
  48             : "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero)
  49             : "cc");
  50 }
  51
  52 static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
  53 {
  54         unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  55         unsigned long offset = vaddr & (PAGE_SIZE - 1);
  56         unsigned long to;
  57
  58         set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
  59         to = va + offset;
  60         flush_icache_range(to, to + len);
  61 }
  62
  63 void flush_cache_mm(struct mm_struct *mm)
  64 {
  65         if (cache_is_vivt()) {
  66                 vivt_flush_cache_mm(mm);
  67                 return;
  68         }
  69
  70         if (cache_is_vipt_aliasing()) {
  71                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  72                 "       mcr     p15, 0, %0, c7, c10, 4"
  73                     :
  74                     : "r" (0)
  75                     : "cc");
  76         }
  77 }
  78
  79 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  80 {
  81         if (cache_is_vivt()) {
  82                 vivt_flush_cache_range(vma, start, end);
  83                 return;
  84         }
  85
  86         if (cache_is_vipt_aliasing()) {
  87                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  88                 "       mcr     p15, 0, %0, c7, c10, 4"
  89                     :
  90                     : "r" (0)
  91                     : "cc");
  92         }
  93
  94         if (vma->vm_flags & VM_EXEC)
  95                 __flush_icache_all();
  96 }
  97
  98 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  99 {
 100         if (cache_is_vivt()) {
 101                 vivt_flush_cache_page(vma, user_addr, pfn);
 102                 return;
 103         }
 104
 105         if (cache_is_vipt_aliasing()) {
 106                 flush_pfn_alias(pfn, user_addr);
 107                 __flush_icache_all();
 108         }
 109
 110         if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
 111                 __flush_icache_all();
 112 }
 113
 114 #else
 115 #define flush_pfn_alias(pfn,vaddr)              do { } while (0)
 116 #define flush_icache_alias(pfn,vaddr,len)       do { } while (0)
 117 #endif
 118
 119 #define FLAG_PA_IS_EXEC 1
 120 #define FLAG_PA_CORE_IN_MM 2
 121
 122 static void flush_ptrace_access_other(void *args)
 123 {
 124         __flush_icache_all();
 125 }
 126
 127 static inline
 128 void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
 129                            unsigned long len, unsigned int flags)
 130 {
 131         if (cache_is_vivt()) {
 132                 if (flags & FLAG_PA_CORE_IN_MM) {
 133                         unsigned long addr = (unsigned long)kaddr;
 134                         __cpuc_coherent_kern_range(addr, addr + len);
 135                 }
 136                 return;
 137         }
 138
 139         if (cache_is_vipt_aliasing()) {
 140                 flush_pfn_alias(page_to_pfn(page), uaddr);
 141                 __flush_icache_all();
 142                 return;
 143         }
 144
 145         /* VIPT non-aliasing D-cache */
 146         if (flags & FLAG_PA_IS_EXEC) {
 147                 unsigned long addr = (unsigned long)kaddr;
 148                 if (icache_is_vipt_aliasing())
 149                         flush_icache_alias(page_to_pfn(page), uaddr, len);
 150                 else
 151                         __cpuc_coherent_kern_range(addr, addr + len);
 152                 if (cache_ops_need_broadcast())
 153                         smp_call_function(flush_ptrace_access_other,
 154                                           NULL, 1);
 155         }
 156 }
 157
 158 static
 159 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
 160                          unsigned long uaddr, void *kaddr, unsigned long len)
 161 {
 162         unsigned int flags = 0;
 163         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
 164                 flags |= FLAG_PA_CORE_IN_MM;
 165         if (vma->vm_flags & VM_EXEC)
 166                 flags |= FLAG_PA_IS_EXEC;
 167         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 168 }
 169
 170 void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
 171                              void *kaddr, unsigned long len)
 172 {
 173         unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
 174
 175         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 176 }
 177
 178 /*
 179  * Copy user data from/to a page which is mapped into a different
 180  * processes address space.  Really, we want to allow our "user
 181  * space" model to handle this.
 182  *
 183  * Note that this code needs to run on the current CPU.
 184  */
 185 void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 186                        unsigned long uaddr, void *dst, const void *src,
 187                        unsigned long len)
 188 {
 189 #ifdef CONFIG_SMP
 190         preempt_disable();
 191 #endif
 192         memcpy(dst, src, len);
 193         flush_ptrace_access(vma, page, uaddr, dst, len);
 194 #ifdef CONFIG_SMP
 195         preempt_enable();
 196 #endif
 197 }
 198
 199 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 200 {
 201         /*
 202          * Writeback any data associated with the kernel mapping of this
 203          * page.  This ensures that data in the physical page is mutually
 204          * coherent with the kernels mapping.
 205          */
 206         if (!PageHighMem(page)) {
 207                 __cpuc_flush_dcache_area(page_address(page), page_size(page));
 208         } else {
 209                 unsigned long i;
 210                 if (cache_is_vipt_nonaliasing()) {
 211                         for (i = 0; i < compound_nr(page); i++) {
 212                                 void *addr = kmap_atomic(page + i);
 213                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 214                                 kunmap_atomic(addr);
 215                         }
 216                 } else {
 217                         for (i = 0; i < compound_nr(page); i++) {
 218                                 void *addr = kmap_high_get(page + i);
 219                                 if (addr) {
 220                                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 221                                         kunmap_high(page + i);
 222                                 }
 223                         }
 224                 }
 225         }
 226
 227         /*
 228          * If this is a page cache page, and we have an aliasing VIPT cache,
 229          * we only need to do one flush - which would be at the relevant
 230          * userspace colour, which is congruent with page->index.
 231          */
 232         if (mapping && cache_is_vipt_aliasing())
 233                 flush_pfn_alias(page_to_pfn(page),
 234                                 page->index << PAGE_SHIFT);
 235 }
 236
 237 static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
 238 {
 239         struct mm_struct *mm = current->active_mm;
 240         struct vm_area_struct *mpnt;
 241         pgoff_t pgoff;
 242
 243         /*
 244          * There are possible user space mappings of this page:
 245          * - VIVT cache: we need to also write back and invalidate all user
 246          *   data in the current VM view associated with this page.
 247          * - aliasing VIPT: we only need to find one mapping of this page.
 248          */
 249         pgoff = page->index;
 250
 251         flush_dcache_mmap_lock(mapping);
 252         vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
 253                 unsigned long offset;
 254
 255                 /*
 256                  * If this VMA is not in our MM, we can ignore it.
 257                  */
 258                 if (mpnt->vm_mm != mm)
 259                         continue;
 260                 if (!(mpnt->vm_flags & VM_MAYSHARE))
 261                         continue;
 262                 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
 263                 flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
 264         }
 265         flush_dcache_mmap_unlock(mapping);
 266 }
 267
 268 #if __LINUX_ARM_ARCH__ >= 6
 269 void __sync_icache_dcache(pte_t pteval)
 270 {
 271         unsigned long pfn;
 272         struct page *page;
 273         struct address_space *mapping;
 274
 275         if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
 276                 /* only flush non-aliasing VIPT caches for exec mappings */
 277                 return;
 278         pfn = pte_pfn(pteval);
 279         if (!pfn_valid(pfn))
 280                 return;
 281
 282         page = pfn_to_page(pfn);
 283         if (cache_is_vipt_aliasing())
 284                 mapping = page_mapping_file(page);
 285         else
 286                 mapping = NULL;
 287
 288         if (!test_and_set_bit(PG_dcache_clean, &page->flags))
 289                 __flush_dcache_page(mapping, page);
 290
 291         if (pte_exec(pteval))
 292                 __flush_icache_all();
 293 }
 294 #endif
 295
 296 /*
 297  * Ensure cache coherency between kernel mapping and userspace mapping
 298  * of this page.
 299  *
 300  * We have three cases to consider:
 301  *  - VIPT non-aliasing cache: fully coherent so nothing required.
 302  *  - VIVT: fully aliasing, so we need to handle every alias in our
 303  *          current VM view.
 304  *  - VIPT aliasing: need to handle one alias in our current VM view.
 305  *
 306  * If we need to handle aliasing:
 307  *  If the page only exists in the page cache and there are no user
 308  *  space mappings, we can be lazy and remember that we may have dirty
 309  *  kernel cache lines for later.  Otherwise, we assume we have
 310  *  aliasing mappings.
 311  *
 312  * Note that we disable the lazy flush for SMP configurations where
 313  * the cache maintenance operations are not automatically broadcasted.
 314  */
 315 void flush_dcache_page(struct page *page)
 316 {
 317         struct address_space *mapping;
 318
 319         /*
 320          * The zero page is never written to, so never has any dirty
 321          * cache lines, and therefore never needs to be flushed.
 322          */
 323         if (page == ZERO_PAGE(0))
 324                 return;
 325
 326         if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
 327                 if (test_bit(PG_dcache_clean, &page->flags))
 328                         clear_bit(PG_dcache_clean, &page->flags);
 329                 return;
 330         }
 331
 332         mapping = page_mapping_file(page);
 333
 334         if (!cache_ops_need_broadcast() &&
 335             mapping && !page_mapcount(page))
 336                 clear_bit(PG_dcache_clean, &page->flags);
 337         else {
 338                 __flush_dcache_page(mapping, page);
 339                 if (mapping && cache_is_vivt())
 340                         __flush_dcache_aliases(mapping, page);
 341                 else if (mapping)
 342                         __flush_icache_all();
 343                 set_bit(PG_dcache_clean, &page->flags);
 344         }
 345 }
 346 EXPORT_SYMBOL(flush_dcache_page);
 347
 348 /*
 349  * Ensure cache coherency for the kernel mapping of this page. We can
 350  * assume that the page is pinned via kmap.
 351  *
 352  * If the page only exists in the page cache and there are no user
 353  * space mappings, this is a no-op since the page was already marked
 354  * dirty at creation.  Otherwise, we need to flush the dirty kernel
 355  * cache lines directly.
 356  */
 357 void flush_kernel_dcache_page(struct page *page)
 358 {
 359         if (cache_is_vivt() || cache_is_vipt_aliasing()) {
 360                 struct address_space *mapping;
 361
 362                 mapping = page_mapping_file(page);
 363
 364                 if (!mapping || mapping_mapped(mapping)) {
 365                         void *addr;
 366
 367                         addr = page_address(page);
 368                         /*
 369                          * kmap_atomic() doesn't set the page virtual
 370                          * address for highmem pages, and
 371                          * kunmap_atomic() takes care of cache
 372                          * flushing already.
 373                          */
 374                         if (!IS_ENABLED(CONFIG_HIGHMEM) || addr)
 375                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 376                 }
 377         }
 378 }
 379 EXPORT_SYMBOL(flush_kernel_dcache_page);
 380
 381 /*
 382  * Flush an anonymous page so that users of get_user_pages()
 383  * can safely access the data.  The expected sequence is:
 384  *
 385  *  get_user_pages()
 386  *    -> flush_anon_page
 387  *  memcpy() to/from page
 388  *  if written to page, flush_dcache_page()
 389  */
 390 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
 391 {
 392         unsigned long pfn;
 393
 394         /* VIPT non-aliasing caches need do nothing */
 395         if (cache_is_vipt_nonaliasing())
 396                 return;
 397
 398         /*
 399          * Write back and invalidate userspace mapping.
 400          */
 401         pfn = page_to_pfn(page);
 402         if (cache_is_vivt()) {
 403                 flush_cache_page(vma, vmaddr, pfn);
 404         } else {
 405                 /*
 406                  * For aliasing VIPT, we can flush an alias of the
 407                  * userspace address only.
 408                  */
 409                 flush_pfn_alias(pfn, vmaddr);
 410                 __flush_icache_all();
 411         }
 412
 413         /*
 414          * Invalidate kernel mapping.  No data should be contained
 415          * in this mapping of the page.  FIXME: this is overkill
 416          * since we actually ask for a write-back and invalidate.
 417          */
 418         __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 419 }