arch/mn10300/mm/tlb-smp.c

   1 /* SMP TLB support routines.
   2  *
   3  * Copyright (C) 2006-2008 Panasonic Corporation
   4  * All Rights Reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * version 2 as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  */
  15 #include <linux/interrupt.h>
  16 #include <linux/spinlock.h>
  17 #include <linux/init.h>
  18 #include <linux/jiffies.h>
  19 #include <linux/cpumask.h>
  20 #include <linux/err.h>
  21 #include <linux/kernel.h>
  22 #include <linux/delay.h>
  23 #include <linux/sched.h>
  24 #include <linux/profile.h>
  25 #include <linux/smp.h>
  26 #include <asm/tlbflush.h>
  27 #include <asm/system.h>
  28 #include <asm/bitops.h>
  29 #include <asm/processor.h>
  30 #include <asm/bug.h>
  31 #include <asm/exceptions.h>
  32 #include <asm/hardirq.h>
  33 #include <asm/fpu.h>
  34 #include <asm/mmu_context.h>
  35 #include <asm/thread_info.h>
  36 #include <asm/cpu-regs.h>
  37 #include <asm/intctl-regs.h>
  38
  39 /*
  40  * For flush TLB
  41  */
  42 #define FLUSH_ALL       0xffffffff
  43
  44 static cpumask_t flush_cpumask;
  45 static struct mm_struct *flush_mm;
  46 static unsigned long flush_va;
  47 static DEFINE_SPINLOCK(tlbstate_lock);
  48
  49 DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
  50         &init_mm, 0
  51 };
  52
  53 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
  54                              unsigned long va);
  55 static void do_flush_tlb_all(void *info);
  56
  57 /**
  58  * smp_flush_tlb - Callback to invalidate the TLB.
  59  * @unused: Callback context (ignored).
  60  */
  61 void smp_flush_tlb(void *unused)
  62 {
  63         unsigned long cpu_id;
  64
  65         cpu_id = get_cpu();
  66
  67         if (!cpumask_test_cpu(cpu_id, &flush_cpumask))
  68                 /* This was a BUG() but until someone can quote me the line
  69                  * from the intel manual that guarantees an IPI to multiple
  70                  * CPUs is retried _only_ on the erroring CPUs its staying as a
  71                  * return
  72                  *
  73                  * BUG();
  74                  */
  75                 goto out;
  76
  77         if (flush_va == FLUSH_ALL)
  78                 local_flush_tlb();
  79         else
  80                 local_flush_tlb_page(flush_mm, flush_va);
  81
  82         smp_mb__before_clear_bit();
  83         cpumask_clear_cpu(cpu_id, &flush_cpumask);
  84         smp_mb__after_clear_bit();
  85 out:
  86         put_cpu();
  87 }
  88
  89 /**
  90  * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs
  91  * @cpumask: The list of CPUs to target.
  92  * @mm: The VM context to flush from (if va!=FLUSH_ALL).
  93  * @va: Virtual address to flush or FLUSH_ALL to flush everything.
  94  */
  95 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
  96                              unsigned long va)
  97 {
  98         cpumask_t tmp;
  99
 100         /* A couple of sanity checks (to be removed):
 101          * - mask must not be empty
 102          * - current CPU must not be in mask
 103          * - we do not send IPIs to as-yet unbooted CPUs.
 104          */
 105         BUG_ON(!mm);
 106         BUG_ON(cpumask_empty(&cpumask));
 107         BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask));
 108
 109         cpumask_and(&tmp, &cpumask, cpu_online_mask);
 110         BUG_ON(!cpumask_equal(&cpumask, &tmp));
 111
 112         /* I'm not happy about this global shared spinlock in the MM hot path,
 113          * but we'll see how contended it is.
 114          *
 115          * Temporarily this turns IRQs off, so that lockups are detected by the
 116          * NMI watchdog.
 117          */
 118         spin_lock(&tlbstate_lock);
 119
 120         flush_mm = mm;
 121         flush_va = va;
 122 #if NR_CPUS <= BITS_PER_LONG
 123         atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]);
 124 #else
 125 #error Not supported.
 126 #endif
 127
 128         /* FIXME: if NR_CPUS>=3, change send_IPI_mask */
 129         smp_call_function(smp_flush_tlb, NULL, 1);
 130
 131         while (!cpumask_empty(&flush_cpumask))
 132                 /* Lockup detection does not belong here */
 133                 smp_mb();
 134
 135         flush_mm = NULL;
 136         flush_va = 0;
 137         spin_unlock(&tlbstate_lock);
 138 }
 139
 140 /**
 141  * flush_tlb_mm - Invalidate TLB of specified VM context
 142  * @mm: The VM context to invalidate.
 143  */
 144 void flush_tlb_mm(struct mm_struct *mm)
 145 {
 146         cpumask_t cpu_mask;
 147
 148         preempt_disable();
 149         cpumask_copy(&cpu_mask, mm_cpumask(mm));
 150         cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 151
 152         local_flush_tlb();
 153         if (!cpumask_empty(&cpu_mask))
 154                 flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
 155
 156         preempt_enable();
 157 }
 158
 159 /**
 160  * flush_tlb_current_task - Invalidate TLB of current task
 161  */
 162 void flush_tlb_current_task(void)
 163 {
 164         struct mm_struct *mm = current->mm;
 165         cpumask_t cpu_mask;
 166
 167         preempt_disable();
 168         cpumask_copy(&cpu_mask, mm_cpumask(mm));
 169         cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 170
 171         local_flush_tlb();
 172         if (!cpumask_empty(&cpu_mask))
 173                 flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
 174
 175         preempt_enable();
 176 }
 177
 178 /**
 179  * flush_tlb_page - Invalidate TLB of page
 180  * @vma: The VM context to invalidate the page for.
 181  * @va: The virtual address of the page to invalidate.
 182  */
 183 void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
 184 {
 185         struct mm_struct *mm = vma->vm_mm;
 186         cpumask_t cpu_mask;
 187
 188         preempt_disable();
 189         cpumask_copy(&cpu_mask, mm_cpumask(mm));
 190         cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 191
 192         local_flush_tlb_page(mm, va);
 193         if (!cpumask_empty(&cpu_mask))
 194                 flush_tlb_others(cpu_mask, mm, va);
 195
 196         preempt_enable();
 197 }
 198
 199 /**
 200  * do_flush_tlb_all - Callback to completely invalidate a TLB
 201  * @unused: Callback context (ignored).
 202  */
 203 static void do_flush_tlb_all(void *unused)
 204 {
 205         local_flush_tlb_all();
 206 }
 207
 208 /**
 209  * flush_tlb_all - Completely invalidate TLBs on all CPUs
 210  */
 211 void flush_tlb_all(void)
 212 {
 213         on_each_cpu(do_flush_tlb_all, 0, 1);
 214 }