arch/arm/mm/flush.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/mm/flush.c
   4  *
   5  *  Copyright (C) 1995-2002 Russell King
   6  */
   7 #include <linux/module.h>
   8 #include <linux/mm.h>
   9 #include <linux/pagemap.h>
  10 #include <linux/highmem.h>
  11
  12 #include <asm/cacheflush.h>
  13 #include <asm/cachetype.h>
  14 #include <asm/highmem.h>
  15 #include <asm/smp_plat.h>
  16 #include <asm/tlbflush.h>
  17 #include <linux/hugetlb.h>
  18
  19 #include "mm.h"
  20
  21 #ifdef CONFIG_ARM_HEAVY_MB
  22 void (*soc_mb)(void);
  23
  24 void arm_heavy_mb(void)
  25 {
  26 #ifdef CONFIG_OUTER_CACHE_SYNC
  27         if (outer_cache.sync)
  28                 outer_cache.sync();
  29 #endif
  30         if (soc_mb)
  31                 soc_mb();
  32 }
  33 EXPORT_SYMBOL(arm_heavy_mb);
  34 #endif
  35
  36 #ifdef CONFIG_CPU_CACHE_VIPT
  37
  38 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  39 {
  40         unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  41         const int zero = 0;
  42
  43         set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
  44
  45         asm(    "mcrr   p15, 0, %1, %0, c14\n"
  46         "       mcr     p15, 0, %2, c7, c10, 4"
  47             :
  48             : "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero)
  49             : "cc");
  50 }
  51
  52 static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
  53 {
  54         unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  55         unsigned long offset = vaddr & (PAGE_SIZE - 1);
  56         unsigned long to;
  57
  58         set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
  59         to = va + offset;
  60         flush_icache_range(to, to + len);
  61 }
  62
  63 void flush_cache_mm(struct mm_struct *mm)
  64 {
  65         if (cache_is_vivt()) {
  66                 vivt_flush_cache_mm(mm);
  67                 return;
  68         }
  69
  70         if (cache_is_vipt_aliasing()) {
  71                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  72                 "       mcr     p15, 0, %0, c7, c10, 4"
  73                     :
  74                     : "r" (0)
  75                     : "cc");
  76         }
  77 }
  78
  79 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  80 {
  81         if (cache_is_vivt()) {
  82                 vivt_flush_cache_range(vma, start, end);
  83                 return;
  84         }
  85
  86         if (cache_is_vipt_aliasing()) {
  87                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  88                 "       mcr     p15, 0, %0, c7, c10, 4"
  89                     :
  90                     : "r" (0)
  91                     : "cc");
  92         }
  93
  94         if (vma->vm_flags & VM_EXEC)
  95                 __flush_icache_all();
  96 }
  97
  98 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
  99 {
 100         if (cache_is_vivt()) {
 101                 vivt_flush_cache_page(vma, user_addr, pfn);
 102                 return;
 103         }
 104
 105         if (cache_is_vipt_aliasing()) {
 106                 flush_pfn_alias(pfn, user_addr);
 107                 __flush_icache_all();
 108         }
 109
 110         if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
 111                 __flush_icache_all();
 112 }
 113
 114 #else
 115 #define flush_pfn_alias(pfn,vaddr)              do { } while (0)
 116 #define flush_icache_alias(pfn,vaddr,len)       do { } while (0)
 117 #endif
 118
 119 #define FLAG_PA_IS_EXEC 1
 120 #define FLAG_PA_CORE_IN_MM 2
 121
 122 static void flush_ptrace_access_other(void *args)
 123 {
 124         __flush_icache_all();
 125 }
 126
 127 static inline
 128 void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
 129                            unsigned long len, unsigned int flags)
 130 {
 131         if (cache_is_vivt()) {
 132                 if (flags & FLAG_PA_CORE_IN_MM) {
 133                         unsigned long addr = (unsigned long)kaddr;
 134                         __cpuc_coherent_kern_range(addr, addr + len);
 135                 }
 136                 return;
 137         }
 138
 139         if (cache_is_vipt_aliasing()) {
 140                 flush_pfn_alias(page_to_pfn(page), uaddr);
 141                 __flush_icache_all();
 142                 return;
 143         }
 144
 145         /* VIPT non-aliasing D-cache */
 146         if (flags & FLAG_PA_IS_EXEC) {
 147                 unsigned long addr = (unsigned long)kaddr;
 148                 if (icache_is_vipt_aliasing())
 149                         flush_icache_alias(page_to_pfn(page), uaddr, len);
 150                 else
 151                         __cpuc_coherent_kern_range(addr, addr + len);
 152                 if (cache_ops_need_broadcast())
 153                         smp_call_function(flush_ptrace_access_other,
 154                                           NULL, 1);
 155         }
 156 }
 157
 158 static
 159 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
 160                          unsigned long uaddr, void *kaddr, unsigned long len)
 161 {
 162         unsigned int flags = 0;
 163         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
 164                 flags |= FLAG_PA_CORE_IN_MM;
 165         if (vma->vm_flags & VM_EXEC)
 166                 flags |= FLAG_PA_IS_EXEC;
 167         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 168 }
 169
 170 void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
 171                              void *kaddr, unsigned long len)
 172 {
 173         unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
 174
 175         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 176 }
 177
 178 /*
 179  * Copy user data from/to a page which is mapped into a different
 180  * processes address space.  Really, we want to allow our "user
 181  * space" model to handle this.
 182  *
 183  * Note that this code needs to run on the current CPU.
 184  */
 185 void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 186                        unsigned long uaddr, void *dst, const void *src,
 187                        unsigned long len)
 188 {
 189 #ifdef CONFIG_SMP
 190         preempt_disable();
 191 #endif
 192         memcpy(dst, src, len);
 193         flush_ptrace_access(vma, page, uaddr, dst, len);
 194 #ifdef CONFIG_SMP
 195         preempt_enable();
 196 #endif
 197 }
 198
 199 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 200 {
 201         /*
 202          * Writeback any data associated with the kernel mapping of this
 203          * page.  This ensures that data in the physical page is mutually
 204          * coherent with the kernels mapping.
 205          */
 206         if (!PageHighMem(page)) {
 207                 size_t page_size = PAGE_SIZE << compound_order(page);
 208                 __cpuc_flush_dcache_area(page_address(page), page_size);
 209         } else {
 210                 unsigned long i;
 211                 if (cache_is_vipt_nonaliasing()) {
 212                         for (i = 0; i < (1 << compound_order(page)); i++) {
 213                                 void *addr = kmap_atomic(page + i);
 214                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 215                                 kunmap_atomic(addr);
 216                         }
 217                 } else {
 218                         for (i = 0; i < (1 << compound_order(page)); i++) {
 219                                 void *addr = kmap_high_get(page + i);
 220                                 if (addr) {
 221                                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 222                                         kunmap_high(page + i);
 223                                 }
 224                         }
 225                 }
 226         }
 227
 228         /*
 229          * If this is a page cache page, and we have an aliasing VIPT cache,
 230          * we only need to do one flush - which would be at the relevant
 231          * userspace colour, which is congruent with page->index.
 232          */
 233         if (mapping && cache_is_vipt_aliasing())
 234                 flush_pfn_alias(page_to_pfn(page),
 235                                 page->index << PAGE_SHIFT);
 236 }
 237
 238 static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
 239 {
 240         struct mm_struct *mm = current->active_mm;
 241         struct vm_area_struct *mpnt;
 242         pgoff_t pgoff;
 243
 244         /*
 245          * There are possible user space mappings of this page:
 246          * - VIVT cache: we need to also write back and invalidate all user
 247          *   data in the current VM view associated with this page.
 248          * - aliasing VIPT: we only need to find one mapping of this page.
 249          */
 250         pgoff = page->index;
 251
 252         flush_dcache_mmap_lock(mapping);
 253         vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
 254                 unsigned long offset;
 255
 256                 /*
 257                  * If this VMA is not in our MM, we can ignore it.
 258                  */
 259                 if (mpnt->vm_mm != mm)
 260                         continue;
 261                 if (!(mpnt->vm_flags & VM_MAYSHARE))
 262                         continue;
 263                 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
 264                 flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
 265         }
 266         flush_dcache_mmap_unlock(mapping);
 267 }
 268
 269 #if __LINUX_ARM_ARCH__ >= 6
 270 void __sync_icache_dcache(pte_t pteval)
 271 {
 272         unsigned long pfn;
 273         struct page *page;
 274         struct address_space *mapping;
 275
 276         if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
 277                 /* only flush non-aliasing VIPT caches for exec mappings */
 278                 return;
 279         pfn = pte_pfn(pteval);
 280         if (!pfn_valid(pfn))
 281                 return;
 282
 283         page = pfn_to_page(pfn);
 284         if (cache_is_vipt_aliasing())
 285                 mapping = page_mapping_file(page);
 286         else
 287                 mapping = NULL;
 288
 289         if (!test_and_set_bit(PG_dcache_clean, &page->flags))
 290                 __flush_dcache_page(mapping, page);
 291
 292         if (pte_exec(pteval))
 293                 __flush_icache_all();
 294 }
 295 #endif
 296
 297 /*
 298  * Ensure cache coherency between kernel mapping and userspace mapping
 299  * of this page.
 300  *
 301  * We have three cases to consider:
 302  *  - VIPT non-aliasing cache: fully coherent so nothing required.
 303  *  - VIVT: fully aliasing, so we need to handle every alias in our
 304  *          current VM view.
 305  *  - VIPT aliasing: need to handle one alias in our current VM view.
 306  *
 307  * If we need to handle aliasing:
 308  *  If the page only exists in the page cache and there are no user
 309  *  space mappings, we can be lazy and remember that we may have dirty
 310  *  kernel cache lines for later.  Otherwise, we assume we have
 311  *  aliasing mappings.
 312  *
 313  * Note that we disable the lazy flush for SMP configurations where
 314  * the cache maintenance operations are not automatically broadcasted.
 315  */
 316 void flush_dcache_page(struct page *page)
 317 {
 318         struct address_space *mapping;
 319
 320         /*
 321          * The zero page is never written to, so never has any dirty
 322          * cache lines, and therefore never needs to be flushed.
 323          */
 324         if (page == ZERO_PAGE(0))
 325                 return;
 326
 327         if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
 328                 if (test_bit(PG_dcache_clean, &page->flags))
 329                         clear_bit(PG_dcache_clean, &page->flags);
 330                 return;
 331         }
 332
 333         mapping = page_mapping_file(page);
 334
 335         if (!cache_ops_need_broadcast() &&
 336             mapping && !page_mapcount(page))
 337                 clear_bit(PG_dcache_clean, &page->flags);
 338         else {
 339                 __flush_dcache_page(mapping, page);
 340                 if (mapping && cache_is_vivt())
 341                         __flush_dcache_aliases(mapping, page);
 342                 else if (mapping)
 343                         __flush_icache_all();
 344                 set_bit(PG_dcache_clean, &page->flags);
 345         }
 346 }
 347 EXPORT_SYMBOL(flush_dcache_page);
 348
 349 /*
 350  * Ensure cache coherency for the kernel mapping of this page. We can
 351  * assume that the page is pinned via kmap.
 352  *
 353  * If the page only exists in the page cache and there are no user
 354  * space mappings, this is a no-op since the page was already marked
 355  * dirty at creation.  Otherwise, we need to flush the dirty kernel
 356  * cache lines directly.
 357  */
 358 void flush_kernel_dcache_page(struct page *page)
 359 {
 360         if (cache_is_vivt() || cache_is_vipt_aliasing()) {
 361                 struct address_space *mapping;
 362
 363                 mapping = page_mapping_file(page);
 364
 365                 if (!mapping || mapping_mapped(mapping)) {
 366                         void *addr;
 367
 368                         addr = page_address(page);
 369                         /*
 370                          * kmap_atomic() doesn't set the page virtual
 371                          * address for highmem pages, and
 372                          * kunmap_atomic() takes care of cache
 373                          * flushing already.
 374                          */
 375                         if (!IS_ENABLED(CONFIG_HIGHMEM) || addr)
 376                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 377                 }
 378         }
 379 }
 380 EXPORT_SYMBOL(flush_kernel_dcache_page);
 381
 382 /*
 383  * Flush an anonymous page so that users of get_user_pages()
 384  * can safely access the data.  The expected sequence is:
 385  *
 386  *  get_user_pages()
 387  *    -> flush_anon_page
 388  *  memcpy() to/from page
 389  *  if written to page, flush_dcache_page()
 390  */
 391 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
 392 {
 393         unsigned long pfn;
 394
 395         /* VIPT non-aliasing caches need do nothing */
 396         if (cache_is_vipt_nonaliasing())
 397                 return;
 398
 399         /*
 400          * Write back and invalidate userspace mapping.
 401          */
 402         pfn = page_to_pfn(page);
 403         if (cache_is_vivt()) {
 404                 flush_cache_page(vma, vmaddr, pfn);
 405         } else {
 406                 /*
 407                  * For aliasing VIPT, we can flush an alias of the
 408                  * userspace address only.
 409                  */
 410                 flush_pfn_alias(pfn, vmaddr);
 411                 __flush_icache_all();
 412         }
 413
 414         /*
 415          * Invalidate kernel mapping.  No data should be contained
 416          * in this mapping of the page.  FIXME: this is overkill
 417          * since we actually ask for a write-back and invalidate.
 418          */
 419         __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 420 }