arch/arm/mm/context.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/arch/arm/mm/context.c
   4  *
   5  *  Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
   6  *  Copyright (C) 2012 ARM Limited
   7  *
   8  *  Author: Will Deacon <will.deacon@arm.com>
   9  */
  10 #include <linux/init.h>
  11 #include <linux/sched.h>
  12 #include <linux/mm.h>
  13 #include <linux/smp.h>
  14 #include <linux/percpu.h>
  15
  16 #include <asm/mmu_context.h>
  17 #include <asm/smp_plat.h>
  18 #include <asm/thread_notify.h>
  19 #include <asm/tlbflush.h>
  20 #include <asm/proc-fns.h>
  21
  22 /*
  23  * On ARMv6, we have the following structure in the Context ID:
  24  *
  25  * 31                         7          0
  26  * +-------------------------+-----------+
  27  * |      process ID         |   ASID    |
  28  * +-------------------------+-----------+
  29  * |              context ID             |
  30  * +-------------------------------------+
  31  *
  32  * The ASID is used to tag entries in the CPU caches and TLBs.
  33  * The context ID is used by debuggers and trace logic, and
  34  * should be unique within all running processes.
  35  *
  36  * In big endian operation, the two 32 bit words are swapped if accessed
  37  * by non-64-bit operations.
  38  */
  39 #define ASID_FIRST_VERSION      (1ULL << ASID_BITS)
  40 #define NUM_USER_ASIDS          ASID_FIRST_VERSION
  41
  42 static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
  43 static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
  44 static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
  45
  46 static DEFINE_PER_CPU(atomic64_t, active_asids);
  47 static DEFINE_PER_CPU(u64, reserved_asids);
  48 static cpumask_t tlb_flush_pending;
  49
  50 #ifdef CONFIG_ARM_ERRATA_798181
  51 void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
  52                              cpumask_t *mask)
  53 {
  54         int cpu;
  55         unsigned long flags;
  56         u64 context_id, asid;
  57
  58         raw_spin_lock_irqsave(&cpu_asid_lock, flags);
  59         context_id = mm->context.id.counter;
  60         for_each_online_cpu(cpu) {
  61                 if (cpu == this_cpu)
  62                         continue;
  63                 /*
  64                  * We only need to send an IPI if the other CPUs are
  65                  * running the same ASID as the one being invalidated.
  66                  */
  67                 asid = per_cpu(active_asids, cpu).counter;
  68                 if (asid == 0)
  69                         asid = per_cpu(reserved_asids, cpu);
  70                 if (context_id == asid)
  71                         cpumask_set_cpu(cpu, mask);
  72         }
  73         raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
  74 }
  75 #endif
  76
  77 #ifdef CONFIG_ARM_LPAE
  78 /*
  79  * With LPAE, the ASID and page tables are updated atomicly, so there is
  80  * no need for a reserved set of tables (the active ASID tracking prevents
  81  * any issues across a rollover).
  82  */
  83 #define cpu_set_reserved_ttbr0()
  84 #else
  85 static void cpu_set_reserved_ttbr0(void)
  86 {
  87         u32 ttb;
  88         /*
  89          * Copy TTBR1 into TTBR0.
  90          * This points at swapper_pg_dir, which contains only global
  91          * entries so any speculative walks are perfectly safe.
  92          */
  93         asm volatile(
  94         "       mrc     p15, 0, %0, c2, c0, 1           @ read TTBR1\n"
  95         "       mcr     p15, 0, %0, c2, c0, 0           @ set TTBR0\n"
  96         : "=r" (ttb));
  97         isb();
  98 }
  99 #endif
 100
 101 #ifdef CONFIG_PID_IN_CONTEXTIDR
 102 static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
 103                                void *t)
 104 {
 105         u32 contextidr;
 106         pid_t pid;
 107         struct thread_info *thread = t;
 108
 109         if (cmd != THREAD_NOTIFY_SWITCH)
 110                 return NOTIFY_DONE;
 111
 112         pid = task_pid_nr(thread->task) << ASID_BITS;
 113         asm volatile(
 114         "       mrc     p15, 0, %0, c13, c0, 1\n"
 115         "       and     %0, %0, %2\n"
 116         "       orr     %0, %0, %1\n"
 117         "       mcr     p15, 0, %0, c13, c0, 1\n"
 118         : "=r" (contextidr), "+r" (pid)
 119         : "I" (~ASID_MASK));
 120         isb();
 121
 122         return NOTIFY_OK;
 123 }
 124
 125 static struct notifier_block contextidr_notifier_block = {
 126         .notifier_call = contextidr_notifier,
 127 };
 128
 129 static int __init contextidr_notifier_init(void)
 130 {
 131         return thread_register_notifier(&contextidr_notifier_block);
 132 }
 133 arch_initcall(contextidr_notifier_init);
 134 #endif
 135
 136 static void flush_context(unsigned int cpu)
 137 {
 138         int i;
 139         u64 asid;
 140
 141         /* Update the list of reserved ASIDs and the ASID bitmap. */
 142         bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 143         for_each_possible_cpu(i) {
 144                 asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
 145                 /*
 146                  * If this CPU has already been through a
 147                  * rollover, but hasn't run another task in
 148                  * the meantime, we must preserve its reserved
 149                  * ASID, as this is the only trace we have of
 150                  * the process it is still running.
 151                  */
 152                 if (asid == 0)
 153                         asid = per_cpu(reserved_asids, i);
 154                 __set_bit(asid & ~ASID_MASK, asid_map);
 155                 per_cpu(reserved_asids, i) = asid;
 156         }
 157
 158         /* Queue a TLB invalidate and flush the I-cache if necessary. */
 159         cpumask_setall(&tlb_flush_pending);
 160
 161         if (icache_is_vivt_asid_tagged())
 162                 __flush_icache_all();
 163 }
 164
 165 static bool check_update_reserved_asid(u64 asid, u64 newasid)
 166 {
 167         int cpu;
 168         bool hit = false;
 169
 170         /*
 171          * Iterate over the set of reserved ASIDs looking for a match.
 172          * If we find one, then we can update our mm to use newasid
 173          * (i.e. the same ASID in the current generation) but we can't
 174          * exit the loop early, since we need to ensure that all copies
 175          * of the old ASID are updated to reflect the mm. Failure to do
 176          * so could result in us missing the reserved ASID in a future
 177          * generation.
 178          */
 179         for_each_possible_cpu(cpu) {
 180                 if (per_cpu(reserved_asids, cpu) == asid) {
 181                         hit = true;
 182                         per_cpu(reserved_asids, cpu) = newasid;
 183                 }
 184         }
 185
 186         return hit;
 187 }
 188
 189 static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 190 {
 191         static u32 cur_idx = 1;
 192         u64 asid = atomic64_read(&mm->context.id);
 193         u64 generation = atomic64_read(&asid_generation);
 194
 195         if (asid != 0) {
 196                 u64 newasid = generation | (asid & ~ASID_MASK);
 197
 198                 /*
 199                  * If our current ASID was active during a rollover, we
 200                  * can continue to use it and this was just a false alarm.
 201                  */
 202                 if (check_update_reserved_asid(asid, newasid))
 203                         return newasid;
 204
 205                 /*
 206                  * We had a valid ASID in a previous life, so try to re-use
 207                  * it if possible.,
 208                  */
 209                 asid &= ~ASID_MASK;
 210                 if (!__test_and_set_bit(asid, asid_map))
 211                         return newasid;
 212         }
 213
 214         /*
 215          * Allocate a free ASID. If we can't find one, take a note of the
 216          * currently active ASIDs and mark the TLBs as requiring flushes.
 217          * We always count from ASID #1, as we reserve ASID #0 to switch
 218          * via TTBR0 and to avoid speculative page table walks from hitting
 219          * in any partial walk caches, which could be populated from
 220          * overlapping level-1 descriptors used to map both the module
 221          * area and the userspace stack.
 222          */
 223         asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
 224         if (asid == NUM_USER_ASIDS) {
 225                 generation = atomic64_add_return(ASID_FIRST_VERSION,
 226                                                  &asid_generation);
 227                 flush_context(cpu);
 228                 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
 229         }
 230
 231         __set_bit(asid, asid_map);
 232         cur_idx = asid;
 233         cpumask_clear(mm_cpumask(mm));
 234         return asid | generation;
 235 }
 236
 237 void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
 238 {
 239         unsigned long flags;
 240         unsigned int cpu = smp_processor_id();
 241         u64 asid;
 242
 243         if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
 244                 __check_vmalloc_seq(mm);
 245
 246         /*
 247          * We cannot update the pgd and the ASID atomicly with classic
 248          * MMU, so switch exclusively to global mappings to avoid
 249          * speculative page table walking with the wrong TTBR.
 250          */
 251         cpu_set_reserved_ttbr0();
 252
 253         asid = atomic64_read(&mm->context.id);
 254         if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
 255             && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
 256                 goto switch_mm_fastpath;
 257
 258         raw_spin_lock_irqsave(&cpu_asid_lock, flags);
 259         /* Check that our ASID belongs to the current generation. */
 260         asid = atomic64_read(&mm->context.id);
 261         if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
 262                 asid = new_context(mm, cpu);
 263                 atomic64_set(&mm->context.id, asid);
 264         }
 265
 266         if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
 267                 local_flush_bp_all();
 268                 local_flush_tlb_all();
 269         }
 270
 271         atomic64_set(&per_cpu(active_asids, cpu), asid);
 272         cpumask_set_cpu(cpu, mm_cpumask(mm));
 273         raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 274
 275 switch_mm_fastpath:
 276         cpu_switch_mm(mm->pgd, mm);
 277 }