arch/arm/kernel/smp_tlb.c

   1 /*
   2  *  linux/arch/arm/kernel/smp_tlb.c
   3  *
   4  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/preempt.h>
  11 #include <linux/smp.h>
  12
  13 #include <asm/smp_plat.h>
  14 #include <asm/tlbflush.h>
  15 #include <asm/mmu_context.h>
  16
  17 /**********************************************************************/
  18
  19 /*
  20  * TLB operations
  21  */
  22 struct tlb_args {
  23         struct vm_area_struct *ta_vma;
  24         unsigned long ta_start;
  25         unsigned long ta_end;
  26 };
  27
  28 static inline void ipi_flush_tlb_all(void *ignored)
  29 {
  30         local_flush_tlb_all();
  31 }
  32
  33 static inline void ipi_flush_tlb_mm(void *arg)
  34 {
  35         struct mm_struct *mm = (struct mm_struct *)arg;
  36
  37         local_flush_tlb_mm(mm);
  38 }
  39
  40 static inline void ipi_flush_tlb_page(void *arg)
  41 {
  42         struct tlb_args *ta = (struct tlb_args *)arg;
  43
  44         local_flush_tlb_page(ta->ta_vma, ta->ta_start);
  45 }
  46
  47 static inline void ipi_flush_tlb_kernel_page(void *arg)
  48 {
  49         struct tlb_args *ta = (struct tlb_args *)arg;
  50
  51         local_flush_tlb_kernel_page(ta->ta_start);
  52 }
  53
  54 static inline void ipi_flush_tlb_range(void *arg)
  55 {
  56         struct tlb_args *ta = (struct tlb_args *)arg;
  57
  58         local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
  59 }
  60
  61 static inline void ipi_flush_tlb_kernel_range(void *arg)
  62 {
  63         struct tlb_args *ta = (struct tlb_args *)arg;
  64
  65         local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
  66 }
  67
  68 static inline void ipi_flush_bp_all(void *ignored)
  69 {
  70         local_flush_bp_all();
  71 }
  72
  73 #ifdef CONFIG_ARM_ERRATA_798181
  74 static int erratum_a15_798181(void)
  75 {
  76         unsigned int midr = read_cpuid_id();
  77
  78         /* Cortex-A15 r0p0..r3p2 affected */
  79         if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
  80                 return 0;
  81         return 1;
  82 }
  83 #else
  84 static int erratum_a15_798181(void)
  85 {
  86         return 0;
  87 }
  88 #endif
  89
  90 static void ipi_flush_tlb_a15_erratum(void *arg)
  91 {
  92         dmb();
  93 }
  94
  95 static void broadcast_tlb_a15_erratum(void)
  96 {
  97         if (!erratum_a15_798181())
  98                 return;
  99
 100         dummy_flush_tlb_a15_erratum();
 101         smp_call_function(ipi_flush_tlb_a15_erratum, NULL, 1);
 102 }
 103
 104 static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
 105 {
 106         int cpu, this_cpu;
 107         cpumask_t mask = { CPU_BITS_NONE };
 108
 109         if (!erratum_a15_798181())
 110                 return;
 111
 112         dummy_flush_tlb_a15_erratum();
 113         this_cpu = get_cpu();
 114         for_each_online_cpu(cpu) {
 115                 if (cpu == this_cpu)
 116                         continue;
 117                 /*
 118                  * We only need to send an IPI if the other CPUs are running
 119                  * the same ASID as the one being invalidated. There is no
 120                  * need for locking around the active_asids check since the
 121                  * switch_mm() function has at least one dmb() (as required by
 122                  * this workaround) in case a context switch happens on
 123                  * another CPU after the condition below.
 124                  */
 125                 if (atomic64_read(&mm->context.id) ==
 126                     atomic64_read(&per_cpu(active_asids, cpu)))
 127                         cpumask_set_cpu(cpu, &mask);
 128         }
 129         smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
 130         put_cpu();
 131 }
 132
 133 void flush_tlb_all(void)
 134 {
 135         if (tlb_ops_need_broadcast())
 136                 on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 137         else
 138                 local_flush_tlb_all();
 139         broadcast_tlb_a15_erratum();
 140 }
 141
 142 void flush_tlb_mm(struct mm_struct *mm)
 143 {
 144         if (tlb_ops_need_broadcast())
 145                 on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
 146         else
 147                 local_flush_tlb_mm(mm);
 148         broadcast_tlb_mm_a15_erratum(mm);
 149 }
 150
 151 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
 152 {
 153         if (tlb_ops_need_broadcast()) {
 154                 struct tlb_args ta;
 155                 ta.ta_vma = vma;
 156                 ta.ta_start = uaddr;
 157                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page,
 158                                         &ta, 1);
 159         } else
 160                 local_flush_tlb_page(vma, uaddr);
 161         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 162 }
 163
 164 void flush_tlb_kernel_page(unsigned long kaddr)
 165 {
 166         if (tlb_ops_need_broadcast()) {
 167                 struct tlb_args ta;
 168                 ta.ta_start = kaddr;
 169                 on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
 170         } else
 171                 local_flush_tlb_kernel_page(kaddr);
 172         broadcast_tlb_a15_erratum();
 173 }
 174
 175 void flush_tlb_range(struct vm_area_struct *vma,
 176                      unsigned long start, unsigned long end)
 177 {
 178         if (tlb_ops_need_broadcast()) {
 179                 struct tlb_args ta;
 180                 ta.ta_vma = vma;
 181                 ta.ta_start = start;
 182                 ta.ta_end = end;
 183                 on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range,
 184                                         &ta, 1);
 185         } else
 186                 local_flush_tlb_range(vma, start, end);
 187         broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 188 }
 189
 190 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 191 {
 192         if (tlb_ops_need_broadcast()) {
 193                 struct tlb_args ta;
 194                 ta.ta_start = start;
 195                 ta.ta_end = end;
 196                 on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
 197         } else
 198                 local_flush_tlb_kernel_range(start, end);
 199         broadcast_tlb_a15_erratum();
 200 }
 201
 202 void flush_bp_all(void)
 203 {
 204         if (tlb_ops_need_broadcast())
 205                 on_each_cpu(ipi_flush_bp_all, NULL, 1);
 206         else
 207                 local_flush_bp_all();
 208 }