arch/blackfin/kernel/ipipe.c

   1 /* -*- linux-c -*-
   2  * linux/arch/blackfin/kernel/ipipe.c
   3  *
   4  * Copyright (C) 2005-2007 Philippe Gerum.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
   9  * USA; either version 2 of the License, or (at your option) any later
  10  * version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  20  *
  21  * Architecture-dependent I-pipe support for the Blackfin.
  22  */
  23
  24 #include <linux/kernel.h>
  25 #include <linux/sched.h>
  26 #include <linux/module.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/percpu.h>
  29 #include <linux/bitops.h>
  30 #include <linux/errno.h>
  31 #include <linux/kthread.h>
  32 #include <linux/unistd.h>
  33 #include <linux/io.h>
  34 #include <asm/system.h>
  35 #include <asm/atomic.h>
  36 #include <asm/irq_handler.h>
  37
  38 DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
  39
  40 asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs);
  41
  42 static void __ipipe_no_irqtail(void);
  43
  44 unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail;
  45 EXPORT_SYMBOL(__ipipe_irq_tail_hook);
  46
  47 unsigned long __ipipe_core_clock;
  48 EXPORT_SYMBOL(__ipipe_core_clock);
  49
  50 unsigned long __ipipe_freq_scale;
  51 EXPORT_SYMBOL(__ipipe_freq_scale);
  52
  53 atomic_t __ipipe_irq_lvdepth[IVG15 + 1];
  54
  55 unsigned long __ipipe_irq_lvmask = bfin_no_irqs;
  56 EXPORT_SYMBOL(__ipipe_irq_lvmask);
  57
  58 static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
  59 {
  60         desc->ipipe_ack(irq, desc);
  61 }
  62
  63 /*
  64  * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
  65  * interrupts are off, and secondary CPUs are still lost in space.
  66  */
  67 void __ipipe_enable_pipeline(void)
  68 {
  69         unsigned irq;
  70
  71         __ipipe_core_clock = get_cclk(); /* Fetch this once. */
  72         __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock;
  73
  74         for (irq = 0; irq < NR_IRQS; ++irq)
  75                 ipipe_virtualize_irq(ipipe_root_domain,
  76                                      irq,
  77                                      (ipipe_irq_handler_t)&asm_do_IRQ,
  78                                      NULL,
  79                                      &__ipipe_ack_irq,
  80                                      IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
  81 }
  82
  83 /*
  84  * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
  85  * interrupt protection log is maintained here for each domain. Hw
  86  * interrupts are masked on entry.
  87  */
  88 void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs)
  89 {
  90         struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
  91         struct ipipe_domain *this_domain, *next_domain;
  92         struct list_head *head, *pos;
  93         struct ipipe_irqdesc *idesc;
  94         int m_ack, s = -1;
  95
  96         /*
  97          * Software-triggered IRQs do not need any ack.  The contents
  98          * of the register frame should only be used when processing
  99          * the timer interrupt, but not for handling any other
 100          * interrupt.
 101          */
 102         m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR);
 103         this_domain = __ipipe_current_domain;
 104         idesc = &this_domain->irqs[irq];
 105
 106         if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control)))
 107                 head = &this_domain->p_link;
 108         else {
 109                 head = __ipipe_pipeline.next;
 110                 next_domain = list_entry(head, struct ipipe_domain, p_link);
 111                 idesc = &next_domain->irqs[irq];
 112                 if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) {
 113                         if (!m_ack && idesc->acknowledge != NULL)
 114                                 idesc->acknowledge(irq, irq_to_desc(irq));
 115                         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
 116                                 s = __test_and_set_bit(IPIPE_STALL_FLAG,
 117                                                        &p->status);
 118                         __ipipe_dispatch_wired(next_domain, irq);
 119                         goto out;
 120                 }
 121         }
 122
 123         /* Ack the interrupt. */
 124
 125         pos = head;
 126         while (pos != &__ipipe_pipeline) {
 127                 next_domain = list_entry(pos, struct ipipe_domain, p_link);
 128                 idesc = &next_domain->irqs[irq];
 129                 if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) {
 130                         __ipipe_set_irq_pending(next_domain, irq);
 131                         if (!m_ack && idesc->acknowledge != NULL) {
 132                                 idesc->acknowledge(irq, irq_to_desc(irq));
 133                                 m_ack = 1;
 134                         }
 135                 }
 136                 if (!test_bit(IPIPE_PASS_FLAG, &idesc->control))
 137                         break;
 138                 pos = next_domain->p_link.next;
 139         }
 140
 141         /*
 142          * Now walk the pipeline, yielding control to the highest
 143          * priority domain that has pending interrupt(s) or
 144          * immediately to the current domain if the interrupt has been
 145          * marked as 'sticky'. This search does not go beyond the
 146          * current domain in the pipeline. We also enforce the
 147          * additional root stage lock (blackfin-specific).
 148          */
 149         if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status))
 150                 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
 151
 152         /*
 153          * If the interrupt preempted the head domain, then do not
 154          * even try to walk the pipeline, unless an interrupt is
 155          * pending for it.
 156          */
 157         if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
 158             !__ipipe_ipending_p(ipipe_head_cpudom_ptr()))
 159                 goto out;
 160
 161         __ipipe_walk_pipeline(head);
 162 out:
 163         if (!s)
 164                 __clear_bit(IPIPE_STALL_FLAG, &p->status);
 165 }
 166
 167 void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 168 {
 169         struct irq_desc *desc = irq_to_desc(irq);
 170         int prio = __ipipe_get_irq_priority(irq);
 171
 172         desc->depth = 0;
 173         if (ipd != &ipipe_root &&
 174             atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1)
 175                 __set_bit(prio, &__ipipe_irq_lvmask);
 176 }
 177 EXPORT_SYMBOL(__ipipe_enable_irqdesc);
 178
 179 void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
 180 {
 181         int prio = __ipipe_get_irq_priority(irq);
 182
 183         if (ipd != &ipipe_root &&
 184             atomic_dec_and_test(&__ipipe_irq_lvdepth[prio]))
 185                 __clear_bit(prio, &__ipipe_irq_lvmask);
 186 }
 187 EXPORT_SYMBOL(__ipipe_disable_irqdesc);
 188
 189 asmlinkage int __ipipe_syscall_root(struct pt_regs *regs)
 190 {
 191         struct ipipe_percpu_domain_data *p;
 192         void (*hook)(void);
 193         int ret;
 194
 195         WARN_ON_ONCE(irqs_disabled_hw());
 196
 197         /*
 198          * We need to run the IRQ tail hook each time we intercept a
 199          * syscall, because we know that important operations might be
 200          * pending there (e.g. Xenomai deferred rescheduling).
 201          */
 202         hook = (__typeof__(hook))__ipipe_irq_tail_hook;
 203         hook();
 204
 205         /*
 206          * This routine either returns:
 207          * 0 -- if the syscall is to be passed to Linux;
 208          * >0 -- if the syscall should not be passed to Linux, and no
 209          * tail work should be performed;
 210          * <0 -- if the syscall should not be passed to Linux but the
 211          * tail work has to be performed (for handling signals etc).
 212          */
 213
 214         if (!__ipipe_syscall_watched_p(current, regs->orig_p0) ||
 215             !__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
 216                 return 0;
 217
 218         ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
 219
 220         hard_local_irq_disable();
 221
 222         /*
 223          * This is the end of the syscall path, so we may
 224          * safely assume a valid Linux task stack here.
 225          */
 226         if (current->ipipe_flags & PF_EVTRET) {
 227                 current->ipipe_flags &= ~PF_EVTRET;
 228                 __ipipe_dispatch_event(IPIPE_EVENT_RETURN, regs);
 229         }
 230
 231         if (!__ipipe_root_domain_p)
 232                 ret = -1;
 233         else {
 234                 p = ipipe_root_cpudom_ptr();
 235                 if (__ipipe_ipending_p(p))
 236                         __ipipe_sync_pipeline();
 237         }
 238
 239         hard_local_irq_enable();
 240
 241         return -ret;
 242 }
 243
 244 static void __ipipe_no_irqtail(void)
 245 {
 246 }
 247
 248 int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
 249 {
 250         info->sys_nr_cpus = num_online_cpus();
 251         info->sys_cpu_freq = ipipe_cpu_freq();
 252         info->sys_hrtimer_irq = IPIPE_TIMER_IRQ;
 253         info->sys_hrtimer_freq = __ipipe_core_clock;
 254         info->sys_hrclock_freq = __ipipe_core_clock;
 255
 256         return 0;
 257 }
 258
 259 /*
 260  * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
 261  * just like if it has been actually received from a hw source. Also
 262  * works for virtual interrupts.
 263  */
 264 int ipipe_trigger_irq(unsigned irq)
 265 {
 266         unsigned long flags;
 267
 268 #ifdef CONFIG_IPIPE_DEBUG
 269         if (irq >= IPIPE_NR_IRQS ||
 270             (ipipe_virtual_irq_p(irq)
 271              && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
 272                 return -EINVAL;
 273 #endif
 274
 275         flags = hard_local_irq_save();
 276         __ipipe_handle_irq(irq, NULL);
 277         hard_local_irq_restore(flags);
 278
 279         return 1;
 280 }
 281
 282 asmlinkage void __ipipe_sync_root(void)
 283 {
 284         void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook;
 285         struct ipipe_percpu_domain_data *p;
 286         unsigned long flags;
 287
 288         BUG_ON(irqs_disabled());
 289
 290         flags = hard_local_irq_save();
 291
 292         if (irq_tail_hook)
 293                 irq_tail_hook();
 294
 295         clear_thread_flag(TIF_IRQ_SYNC);
 296
 297         p = ipipe_root_cpudom_ptr();
 298         if (__ipipe_ipending_p(p))
 299                 __ipipe_sync_pipeline();
 300
 301         hard_local_irq_restore(flags);
 302 }
 303
 304 void ___ipipe_sync_pipeline(void)
 305 {
 306         if (__ipipe_root_domain_p &&
 307             test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status)))
 308                 return;
 309
 310         __ipipe_sync_stage();
 311 }
 312
 313 void __ipipe_disable_root_irqs_hw(void)
 314 {
 315         /*
 316          * This code is called by the ins{bwl} routines (see
 317          * arch/blackfin/lib/ins.S), which are heavily used by the
 318          * network stack. It masks all interrupts but those handled by
 319          * non-root domains, so that we keep decent network transfer
 320          * rates for Linux without inducing pathological jitter for
 321          * the real-time domain.
 322          */
 323         bfin_sti(__ipipe_irq_lvmask);
 324         __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
 325 }
 326
 327 void __ipipe_enable_root_irqs_hw(void)
 328 {
 329         __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
 330         bfin_sti(bfin_irq_flags);
 331 }
 332
 333 /*
 334  * We could use standard atomic bitops in the following root status
 335  * manipulation routines, but let's prepare for SMP support in the
 336  * same move, preventing CPU migration as required.
 337  */
 338 void __ipipe_stall_root(void)
 339 {
 340         unsigned long *p, flags;
 341
 342         flags = hard_local_irq_save();
 343         p = &__ipipe_root_status;
 344         __set_bit(IPIPE_STALL_FLAG, p);
 345         hard_local_irq_restore(flags);
 346 }
 347 EXPORT_SYMBOL(__ipipe_stall_root);
 348
 349 unsigned long __ipipe_test_and_stall_root(void)
 350 {
 351         unsigned long *p, flags;
 352         int x;
 353
 354         flags = hard_local_irq_save();
 355         p = &__ipipe_root_status;
 356         x = __test_and_set_bit(IPIPE_STALL_FLAG, p);
 357         hard_local_irq_restore(flags);
 358
 359         return x;
 360 }
 361 EXPORT_SYMBOL(__ipipe_test_and_stall_root);
 362
 363 unsigned long __ipipe_test_root(void)
 364 {
 365         const unsigned long *p;
 366         unsigned long flags;
 367         int x;
 368
 369         flags = hard_local_irq_save_smp();
 370         p = &__ipipe_root_status;
 371         x = test_bit(IPIPE_STALL_FLAG, p);
 372         hard_local_irq_restore_smp(flags);
 373
 374         return x;
 375 }
 376 EXPORT_SYMBOL(__ipipe_test_root);
 377
 378 void __ipipe_lock_root(void)
 379 {
 380         unsigned long *p, flags;
 381
 382         flags = hard_local_irq_save();
 383         p = &__ipipe_root_status;
 384         __set_bit(IPIPE_SYNCDEFER_FLAG, p);
 385         hard_local_irq_restore(flags);
 386 }
 387 EXPORT_SYMBOL(__ipipe_lock_root);
 388
 389 void __ipipe_unlock_root(void)
 390 {
 391         unsigned long *p, flags;
 392
 393         flags = hard_local_irq_save();
 394         p = &__ipipe_root_status;
 395         __clear_bit(IPIPE_SYNCDEFER_FLAG, p);
 396         hard_local_irq_restore(flags);
 397 }
 398 EXPORT_SYMBOL(__ipipe_unlock_root);