arch/arm/mm/flush.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/mm/flush.c
   4  *
   5  *  Copyright (C) 1995-2002 Russell King
   6  */
   7 #include <linux/module.h>
   8 #include <linux/mm.h>
   9 #include <linux/pagemap.h>
  10 #include <linux/highmem.h>
  11
  12 #include <asm/cacheflush.h>
  13 #include <asm/cachetype.h>
  14 #include <asm/highmem.h>
  15 #include <asm/smp_plat.h>
  16 #include <asm/tlbflush.h>
  17 #include <linux/hugetlb.h>
  18
  19 #include "mm.h"
  20
  21 #ifdef CONFIG_ARM_HEAVY_MB
  22 void (*soc_mb)(void);
  23
  24 void arm_heavy_mb(void)
  25 {
  26 #ifdef CONFIG_OUTER_CACHE_SYNC
  27         if (outer_cache.sync)
  28                 outer_cache.sync();
  29 #endif
  30         if (soc_mb)
  31                 soc_mb();
  32 }
  33 EXPORT_SYMBOL(arm_heavy_mb);
  34 #endif
  35
  36 #ifdef CONFIG_CPU_CACHE_VIPT
  37
  38 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
  39 {
  40         unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  41         const int zero = 0;
  42
  43         set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
  44
  45         asm(    "mcrr   p15, 0, %1, %0, c14\n"
  46         "       mcr     p15, 0, %2, c7, c10, 4"
  47             :
  48             : "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero)
  49             : "cc");
  50 }
  51
  52 static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
  53 {
  54         unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
  55         unsigned long offset = vaddr & (PAGE_SIZE - 1);
  56         unsigned long to;
  57
  58         set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
  59         to = va + offset;
  60         flush_icache_range(to, to + len);
  61 }
  62
  63 void flush_cache_mm(struct mm_struct *mm)
  64 {
  65         if (cache_is_vivt()) {
  66                 vivt_flush_cache_mm(mm);
  67                 return;
  68         }
  69
  70         if (cache_is_vipt_aliasing()) {
  71                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  72                 "       mcr     p15, 0, %0, c7, c10, 4"
  73                     :
  74                     : "r" (0)
  75                     : "cc");
  76         }
  77 }
  78
  79 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
  80 {
  81         if (cache_is_vivt()) {
  82                 vivt_flush_cache_range(vma, start, end);
  83                 return;
  84         }
  85
  86         if (cache_is_vipt_aliasing()) {
  87                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
  88                 "       mcr     p15, 0, %0, c7, c10, 4"
  89                     :
  90                     : "r" (0)
  91                     : "cc");
  92         }
  93
  94         if (vma->vm_flags & VM_EXEC)
  95                 __flush_icache_all();
  96 }
  97
  98 void flush_cache_pages(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn, unsigned int nr)
  99 {
 100         if (cache_is_vivt()) {
 101                 vivt_flush_cache_pages(vma, user_addr, pfn, nr);
 102                 return;
 103         }
 104
 105         if (cache_is_vipt_aliasing()) {
 106                 flush_pfn_alias(pfn, user_addr);
 107                 __flush_icache_all();
 108         }
 109
 110         if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
 111                 __flush_icache_all();
 112 }
 113
 114 #else
 115 #define flush_pfn_alias(pfn,vaddr)              do { } while (0)
 116 #define flush_icache_alias(pfn,vaddr,len)       do { } while (0)
 117 #endif
 118
 119 #define FLAG_PA_IS_EXEC 1
 120 #define FLAG_PA_CORE_IN_MM 2
 121
 122 static void flush_ptrace_access_other(void *args)
 123 {
 124         __flush_icache_all();
 125 }
 126
 127 static inline
 128 void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
 129                            unsigned long len, unsigned int flags)
 130 {
 131         if (cache_is_vivt()) {
 132                 if (flags & FLAG_PA_CORE_IN_MM) {
 133                         unsigned long addr = (unsigned long)kaddr;
 134                         __cpuc_coherent_kern_range(addr, addr + len);
 135                 }
 136                 return;
 137         }
 138
 139         if (cache_is_vipt_aliasing()) {
 140                 flush_pfn_alias(page_to_pfn(page), uaddr);
 141                 __flush_icache_all();
 142                 return;
 143         }
 144
 145         /* VIPT non-aliasing D-cache */
 146         if (flags & FLAG_PA_IS_EXEC) {
 147                 unsigned long addr = (unsigned long)kaddr;
 148                 if (icache_is_vipt_aliasing())
 149                         flush_icache_alias(page_to_pfn(page), uaddr, len);
 150                 else
 151                         __cpuc_coherent_kern_range(addr, addr + len);
 152                 if (cache_ops_need_broadcast())
 153                         smp_call_function(flush_ptrace_access_other,
 154                                           NULL, 1);
 155         }
 156 }
 157
 158 static
 159 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
 160                          unsigned long uaddr, void *kaddr, unsigned long len)
 161 {
 162         unsigned int flags = 0;
 163         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
 164                 flags |= FLAG_PA_CORE_IN_MM;
 165         if (vma->vm_flags & VM_EXEC)
 166                 flags |= FLAG_PA_IS_EXEC;
 167         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 168 }
 169
 170 void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
 171                              void *kaddr, unsigned long len)
 172 {
 173         unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
 174
 175         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
 176 }
 177
 178 /*
 179  * Copy user data from/to a page which is mapped into a different
 180  * processes address space.  Really, we want to allow our "user
 181  * space" model to handle this.
 182  *
 183  * Note that this code needs to run on the current CPU.
 184  */
 185 void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 186                        unsigned long uaddr, void *dst, const void *src,
 187                        unsigned long len)
 188 {
 189 #ifdef CONFIG_SMP
 190         preempt_disable();
 191 #endif
 192         memcpy(dst, src, len);
 193         flush_ptrace_access(vma, page, uaddr, dst, len);
 194 #ifdef CONFIG_SMP
 195         preempt_enable();
 196 #endif
 197 }
 198
 199 void __flush_dcache_folio(struct address_space *mapping, struct folio *folio)
 200 {
 201         /*
 202          * Writeback any data associated with the kernel mapping of this
 203          * page.  This ensures that data in the physical page is mutually
 204          * coherent with the kernels mapping.
 205          */
 206         if (!folio_test_highmem(folio)) {
 207                 __cpuc_flush_dcache_area(folio_address(folio),
 208                                         folio_size(folio));
 209         } else {
 210                 unsigned long i;
 211                 if (cache_is_vipt_nonaliasing()) {
 212                         for (i = 0; i < folio_nr_pages(folio); i++) {
 213                                 void *addr = kmap_local_folio(folio,
 214                                                                 i * PAGE_SIZE);
 215                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 216                                 kunmap_local(addr);
 217                         }
 218                 } else {
 219                         for (i = 0; i < folio_nr_pages(folio); i++) {
 220                                 void *addr = kmap_high_get(folio_page(folio, i));
 221                                 if (addr) {
 222                                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
 223                                         kunmap_high(folio_page(folio, i));
 224                                 }
 225                         }
 226                 }
 227         }
 228
 229         /*
 230          * If this is a page cache page, and we have an aliasing VIPT cache,
 231          * we only need to do one flush - which would be at the relevant
 232          * userspace colour, which is congruent with page->index.
 233          */
 234         if (mapping && cache_is_vipt_aliasing())
 235                 flush_pfn_alias(folio_pfn(folio), folio_pos(folio));
 236 }
 237
 238 static void __flush_dcache_aliases(struct address_space *mapping, struct folio *folio)
 239 {
 240         struct mm_struct *mm = current->active_mm;
 241         struct vm_area_struct *vma;
 242         pgoff_t pgoff, pgoff_end;
 243
 244         /*
 245          * There are possible user space mappings of this page:
 246          * - VIVT cache: we need to also write back and invalidate all user
 247          *   data in the current VM view associated with this page.
 248          * - aliasing VIPT: we only need to find one mapping of this page.
 249          */
 250         pgoff = folio->index;
 251         pgoff_end = pgoff + folio_nr_pages(folio) - 1;
 252
 253         flush_dcache_mmap_lock(mapping);
 254         vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff_end) {
 255                 unsigned long start, offset, pfn;
 256                 unsigned int nr;
 257
 258                 /*
 259                  * If this VMA is not in our MM, we can ignore it.
 260                  */
 261                 if (vma->vm_mm != mm)
 262                         continue;
 263                 if (!(vma->vm_flags & VM_MAYSHARE))
 264                         continue;
 265
 266                 start = vma->vm_start;
 267                 pfn = folio_pfn(folio);
 268                 nr = folio_nr_pages(folio);
 269                 offset = pgoff - vma->vm_pgoff;
 270                 if (offset > -nr) {
 271                         pfn -= offset;
 272                         nr += offset;
 273                 } else {
 274                         start += offset * PAGE_SIZE;
 275                 }
 276                 if (start + nr * PAGE_SIZE > vma->vm_end)
 277                         nr = (vma->vm_end - start) / PAGE_SIZE;
 278
 279                 flush_cache_pages(vma, start, pfn, nr);
 280         }
 281         flush_dcache_mmap_unlock(mapping);
 282 }
 283
 284 #if __LINUX_ARM_ARCH__ >= 6
 285 void __sync_icache_dcache(pte_t pteval)
 286 {
 287         unsigned long pfn;
 288         struct folio *folio;
 289         struct address_space *mapping;
 290
 291         if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
 292                 /* only flush non-aliasing VIPT caches for exec mappings */
 293                 return;
 294         pfn = pte_pfn(pteval);
 295         if (!pfn_valid(pfn))
 296                 return;
 297
 298         folio = page_folio(pfn_to_page(pfn));
 299         if (folio_test_reserved(folio))
 300                 return;
 301
 302         if (cache_is_vipt_aliasing())
 303                 mapping = folio_flush_mapping(folio);
 304         else
 305                 mapping = NULL;
 306
 307         if (!test_and_set_bit(PG_dcache_clean, &folio->flags))
 308                 __flush_dcache_folio(mapping, folio);
 309
 310         if (pte_exec(pteval))
 311                 __flush_icache_all();
 312 }
 313 #endif
 314
 315 /*
 316  * Ensure cache coherency between kernel mapping and userspace mapping
 317  * of this page.
 318  *
 319  * We have three cases to consider:
 320  *  - VIPT non-aliasing cache: fully coherent so nothing required.
 321  *  - VIVT: fully aliasing, so we need to handle every alias in our
 322  *          current VM view.
 323  *  - VIPT aliasing: need to handle one alias in our current VM view.
 324  *
 325  * If we need to handle aliasing:
 326  *  If the page only exists in the page cache and there are no user
 327  *  space mappings, we can be lazy and remember that we may have dirty
 328  *  kernel cache lines for later.  Otherwise, we assume we have
 329  *  aliasing mappings.
 330  *
 331  * Note that we disable the lazy flush for SMP configurations where
 332  * the cache maintenance operations are not automatically broadcasted.
 333  */
 334 void flush_dcache_folio(struct folio *folio)
 335 {
 336         struct address_space *mapping;
 337
 338         /*
 339          * The zero page is never written to, so never has any dirty
 340          * cache lines, and therefore never needs to be flushed.
 341          */
 342         if (is_zero_pfn(folio_pfn(folio)))
 343                 return;
 344
 345         if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
 346                 if (test_bit(PG_dcache_clean, &folio->flags))
 347                         clear_bit(PG_dcache_clean, &folio->flags);
 348                 return;
 349         }
 350
 351         mapping = folio_flush_mapping(folio);
 352
 353         if (!cache_ops_need_broadcast() &&
 354             mapping && !folio_mapped(folio))
 355                 clear_bit(PG_dcache_clean, &folio->flags);
 356         else {
 357                 __flush_dcache_folio(mapping, folio);
 358                 if (mapping && cache_is_vivt())
 359                         __flush_dcache_aliases(mapping, folio);
 360                 else if (mapping)
 361                         __flush_icache_all();
 362                 set_bit(PG_dcache_clean, &folio->flags);
 363         }
 364 }
 365 EXPORT_SYMBOL(flush_dcache_folio);
 366
 367 void flush_dcache_page(struct page *page)
 368 {
 369         flush_dcache_folio(page_folio(page));
 370 }
 371 EXPORT_SYMBOL(flush_dcache_page);
 372 /*
 373  * Flush an anonymous page so that users of get_user_pages()
 374  * can safely access the data.  The expected sequence is:
 375  *
 376  *  get_user_pages()
 377  *    -> flush_anon_page
 378  *  memcpy() to/from page
 379  *  if written to page, flush_dcache_page()
 380  */
 381 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr);
 382 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
 383 {
 384         unsigned long pfn;
 385
 386         /* VIPT non-aliasing caches need do nothing */
 387         if (cache_is_vipt_nonaliasing())
 388                 return;
 389
 390         /*
 391          * Write back and invalidate userspace mapping.
 392          */
 393         pfn = page_to_pfn(page);
 394         if (cache_is_vivt()) {
 395                 flush_cache_page(vma, vmaddr, pfn);
 396         } else {
 397                 /*
 398                  * For aliasing VIPT, we can flush an alias of the
 399                  * userspace address only.
 400                  */
 401                 flush_pfn_alias(pfn, vmaddr);
 402                 __flush_icache_all();
 403         }
 404
 405         /*
 406          * Invalidate kernel mapping.  No data should be contained
 407          * in this mapping of the page.  FIXME: this is overkill
 408          * since we actually ask for a write-back and invalidate.
 409          */
 410         __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
 411 }