2 * Low-level SPU handling
4 * (C) Copyright IBM Deutschland Entwicklung GmbH 2005
6 * Author: Arnd Bergmann <arndb@de.ibm.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/interrupt.h>
26 #include <linux/list.h>
27 #include <linux/module.h>
28 #include <linux/ptrace.h>
29 #include <linux/slab.h>
30 #include <linux/wait.h>
33 #include <linux/mutex.h>
34 #include <linux/linux_logo.h>
36 #include <asm/spu_priv1.h>
37 #include <asm/spu_csa.h>
41 const struct spu_management_ops
*spu_management_ops
;
42 EXPORT_SYMBOL_GPL(spu_management_ops
);
44 const struct spu_priv1_ops
*spu_priv1_ops
;
45 EXPORT_SYMBOL_GPL(spu_priv1_ops
);
47 struct cbe_spu_info cbe_spu_info
[MAX_NUMNODES
];
48 EXPORT_SYMBOL_GPL(cbe_spu_info
);
51 * The spufs fault-handling code needs to call force_sig_info to raise signals
52 * on DMA errors. Export it here to avoid general kernel-wide access to this
55 EXPORT_SYMBOL_GPL(force_sig_info
);
58 * Protects cbe_spu_info and spu->number.
60 static DEFINE_SPINLOCK(spu_lock
);
63 * List of all spus in the system.
65 * This list is iterated by callers from irq context and callers that
66 * want to sleep. Thus modifications need to be done with both
67 * spu_full_list_lock and spu_full_list_mutex held, while iterating
68 * through it requires either of these locks.
70 * In addition spu_full_list_lock protects all assignmens to
73 static LIST_HEAD(spu_full_list
);
74 static DEFINE_SPINLOCK(spu_full_list_lock
);
75 static DEFINE_MUTEX(spu_full_list_mutex
);
81 void spu_invalidate_slbs(struct spu
*spu
)
83 struct spu_priv2 __iomem
*priv2
= spu
->priv2
;
85 if (spu_mfc_sr1_get(spu
) & MFC_STATE1_RELOCATE_MASK
)
86 out_be64(&priv2
->slb_invalidate_all_W
, 0UL);
88 EXPORT_SYMBOL_GPL(spu_invalidate_slbs
);
90 /* This is called by the MM core when a segment size is changed, to
91 * request a flush of all the SPEs using a given mm
93 void spu_flush_all_slbs(struct mm_struct
*mm
)
98 spin_lock_irqsave(&spu_full_list_lock
, flags
);
99 list_for_each_entry(spu
, &spu_full_list
, full_list
) {
101 spu_invalidate_slbs(spu
);
103 spin_unlock_irqrestore(&spu_full_list_lock
, flags
);
106 /* The hack below stinks... try to do something better one of
107 * these days... Does it even work properly with NR_CPUS == 1 ?
109 static inline void mm_needs_global_tlbie(struct mm_struct
*mm
)
111 int nr
= (NR_CPUS
> 1) ? NR_CPUS
: NR_CPUS
+ 1;
113 /* Global TLBIE broadcast required with SPEs. */
114 __cpus_setall(&mm
->cpu_vm_mask
, nr
);
117 void spu_associate_mm(struct spu
*spu
, struct mm_struct
*mm
)
121 spin_lock_irqsave(&spu_full_list_lock
, flags
);
123 spin_unlock_irqrestore(&spu_full_list_lock
, flags
);
125 mm_needs_global_tlbie(mm
);
127 EXPORT_SYMBOL_GPL(spu_associate_mm
);
129 int spu_64k_pages_available(void)
131 return mmu_psize_defs
[MMU_PAGE_64K
].shift
!= 0;
133 EXPORT_SYMBOL_GPL(spu_64k_pages_available
);
135 static void spu_restart_dma(struct spu
*spu
)
137 struct spu_priv2 __iomem
*priv2
= spu
->priv2
;
139 if (!test_bit(SPU_CONTEXT_SWITCH_PENDING
, &spu
->flags
))
140 out_be64(&priv2
->mfc_control_RW
, MFC_CNTL_RESTART_DMA_COMMAND
);
143 static inline void spu_load_slb(struct spu
*spu
, int slbe
, struct spu_slb
*slb
)
145 struct spu_priv2 __iomem
*priv2
= spu
->priv2
;
147 pr_debug("%s: adding SLB[%d] 0x%016lx 0x%016lx\n",
148 __func__
, slbe
, slb
->vsid
, slb
->esid
);
150 out_be64(&priv2
->slb_index_W
, slbe
);
151 out_be64(&priv2
->slb_vsid_RW
, slb
->vsid
);
152 out_be64(&priv2
->slb_esid_RW
, slb
->esid
);
155 static int __spu_trap_data_seg(struct spu
*spu
, unsigned long ea
)
157 struct mm_struct
*mm
= spu
->mm
;
161 pr_debug("%s\n", __FUNCTION__
);
163 if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE
, &spu
->flags
)) {
164 /* SLBs are pre-loaded for context switch, so
165 * we should never get here!
167 printk("%s: invalid access during switch!\n", __func__
);
170 slb
.esid
= (ea
& ESID_MASK
) | SLB_ESID_V
;
172 switch(REGION_ID(ea
)) {
174 #ifdef CONFIG_PPC_MM_SLICES
175 psize
= get_slice_psize(mm
, ea
);
177 psize
= mm
->context
.user_psize
;
179 slb
.vsid
= (get_vsid(mm
->context
.id
, ea
, MMU_SEGSIZE_256M
)
180 << SLB_VSID_SHIFT
) | SLB_VSID_USER
;
182 case VMALLOC_REGION_ID
:
183 if (ea
< VMALLOC_END
)
184 psize
= mmu_vmalloc_psize
;
186 psize
= mmu_io_psize
;
187 slb
.vsid
= (get_kernel_vsid(ea
, MMU_SEGSIZE_256M
)
188 << SLB_VSID_SHIFT
) | SLB_VSID_KERNEL
;
190 case KERNEL_REGION_ID
:
191 psize
= mmu_linear_psize
;
192 slb
.vsid
= (get_kernel_vsid(ea
, MMU_SEGSIZE_256M
)
193 << SLB_VSID_SHIFT
) | SLB_VSID_KERNEL
;
196 /* Future: support kernel segments so that drivers
199 pr_debug("invalid region access at %016lx\n", ea
);
202 slb
.vsid
|= mmu_psize_defs
[psize
].sllp
;
204 spu_load_slb(spu
, spu
->slb_replace
, &slb
);
207 if (spu
->slb_replace
>= 8)
208 spu
->slb_replace
= 0;
210 spu_restart_dma(spu
);
211 spu
->stats
.slb_flt
++;
215 extern int hash_page(unsigned long ea
, unsigned long access
, unsigned long trap
); //XXX
216 static int __spu_trap_data_map(struct spu
*spu
, unsigned long ea
, u64 dsisr
)
218 pr_debug("%s, %lx, %lx\n", __FUNCTION__
, dsisr
, ea
);
220 /* Handle kernel space hash faults immediately.
221 User hash faults need to be deferred to process context. */
222 if ((dsisr
& MFC_DSISR_PTE_NOT_FOUND
)
223 && REGION_ID(ea
) != USER_REGION_ID
224 && hash_page(ea
, _PAGE_PRESENT
, 0x300) == 0) {
225 spu_restart_dma(spu
);
229 if (test_bit(SPU_CONTEXT_SWITCH_ACTIVE
, &spu
->flags
)) {
230 printk("%s: invalid access during switch!\n", __func__
);
234 spu
->class_0_pending
= 0;
238 spu
->stop_callback(spu
);
243 static void __spu_kernel_slb(void *addr
, struct spu_slb
*slb
)
245 unsigned long ea
= (unsigned long)addr
;
248 if (REGION_ID(ea
) == KERNEL_REGION_ID
)
249 llp
= mmu_psize_defs
[mmu_linear_psize
].sllp
;
251 llp
= mmu_psize_defs
[mmu_virtual_psize
].sllp
;
253 slb
->vsid
= (get_kernel_vsid(ea
, MMU_SEGSIZE_256M
) << SLB_VSID_SHIFT
) |
254 SLB_VSID_KERNEL
| llp
;
255 slb
->esid
= (ea
& ESID_MASK
) | SLB_ESID_V
;
259 * Given an array of @nr_slbs SLB entries, @slbs, return non-zero if the
260 * address @new_addr is present.
262 static inline int __slb_present(struct spu_slb
*slbs
, int nr_slbs
,
265 unsigned long ea
= (unsigned long)new_addr
;
268 for (i
= 0; i
< nr_slbs
; i
++)
269 if (!((slbs
[i
].esid
^ ea
) & ESID_MASK
))
276 * Setup the SPU kernel SLBs, in preparation for a context save/restore. We
277 * need to map both the context save area, and the save/restore code.
279 * Because the lscsa and code may cross segment boundaires, we check to see
280 * if mappings are required for the start and end of each range. We currently
281 * assume that the mappings are smaller that one segment - if not, something
282 * is seriously wrong.
284 void spu_setup_kernel_slbs(struct spu
*spu
, struct spu_lscsa
*lscsa
,
285 void *code
, int code_size
)
287 struct spu_slb slbs
[4];
289 /* start and end addresses of both mappings */
291 lscsa
, (void *)lscsa
+ sizeof(*lscsa
) - 1,
292 code
, code
+ code_size
- 1
295 /* check the set of addresses, and create a new entry in the slbs array
296 * if there isn't already a SLB for that address */
297 for (i
= 0; i
< ARRAY_SIZE(addrs
); i
++) {
298 if (__slb_present(slbs
, nr_slbs
, addrs
[i
]))
301 __spu_kernel_slb(addrs
[i
], &slbs
[nr_slbs
]);
305 /* Add the set of SLBs */
306 for (i
= 0; i
< nr_slbs
; i
++)
307 spu_load_slb(spu
, i
, &slbs
[i
]);
309 EXPORT_SYMBOL_GPL(spu_setup_kernel_slbs
);
312 spu_irq_class_0(int irq
, void *data
)
315 unsigned long stat
, mask
;
319 spin_lock(&spu
->register_lock
);
320 mask
= spu_int_mask_get(spu
, 0);
321 stat
= spu_int_stat_get(spu
, 0) & mask
;
323 spu
->class_0_pending
|= stat
;
324 spu
->dsisr
= spu_mfc_dsisr_get(spu
);
325 spu
->dar
= spu_mfc_dar_get(spu
);
326 spin_unlock(&spu
->register_lock
);
328 spu
->stop_callback(spu
);
330 spu_int_stat_clear(spu
, 0, stat
);
336 spu_irq_class_1(int irq
, void *data
)
339 unsigned long stat
, mask
, dar
, dsisr
;
343 /* atomically read & clear class1 status. */
344 spin_lock(&spu
->register_lock
);
345 mask
= spu_int_mask_get(spu
, 1);
346 stat
= spu_int_stat_get(spu
, 1) & mask
;
347 dar
= spu_mfc_dar_get(spu
);
348 dsisr
= spu_mfc_dsisr_get(spu
);
349 if (stat
& CLASS1_STORAGE_FAULT_INTR
)
350 spu_mfc_dsisr_set(spu
, 0ul);
351 spu_int_stat_clear(spu
, 1, stat
);
352 spin_unlock(&spu
->register_lock
);
353 pr_debug("%s: %lx %lx %lx %lx\n", __FUNCTION__
, mask
, stat
,
356 if (stat
& CLASS1_SEGMENT_FAULT_INTR
)
357 __spu_trap_data_seg(spu
, dar
);
359 if (stat
& CLASS1_STORAGE_FAULT_INTR
)
360 __spu_trap_data_map(spu
, dar
, dsisr
);
362 if (stat
& CLASS1_LS_COMPARE_SUSPEND_ON_GET_INTR
)
365 if (stat
& CLASS1_LS_COMPARE_SUSPEND_ON_PUT_INTR
)
368 return stat
? IRQ_HANDLED
: IRQ_NONE
;
372 spu_irq_class_2(int irq
, void *data
)
377 const int mailbox_intrs
=
378 CLASS2_MAILBOX_THRESHOLD_INTR
| CLASS2_MAILBOX_INTR
;
381 spin_lock(&spu
->register_lock
);
382 stat
= spu_int_stat_get(spu
, 2);
383 mask
= spu_int_mask_get(spu
, 2);
384 /* ignore interrupts we're not waiting for */
387 /* mailbox interrupts are level triggered. mask them now before
389 if (stat
& mailbox_intrs
)
390 spu_int_mask_and(spu
, 2, ~(stat
& mailbox_intrs
));
391 /* acknowledge all interrupts before the callbacks */
392 spu_int_stat_clear(spu
, 2, stat
);
393 spin_unlock(&spu
->register_lock
);
395 pr_debug("class 2 interrupt %d, %lx, %lx\n", irq
, stat
, mask
);
397 if (stat
& CLASS2_MAILBOX_INTR
)
398 spu
->ibox_callback(spu
);
400 if (stat
& CLASS2_SPU_STOP_INTR
)
401 spu
->stop_callback(spu
);
403 if (stat
& CLASS2_SPU_HALT_INTR
)
404 spu
->stop_callback(spu
);
406 if (stat
& CLASS2_SPU_DMA_TAG_GROUP_COMPLETE_INTR
)
407 spu
->mfc_callback(spu
);
409 if (stat
& CLASS2_MAILBOX_THRESHOLD_INTR
)
410 spu
->wbox_callback(spu
);
412 spu
->stats
.class2_intr
++;
413 return stat
? IRQ_HANDLED
: IRQ_NONE
;
416 static int spu_request_irqs(struct spu
*spu
)
420 if (spu
->irqs
[0] != NO_IRQ
) {
421 snprintf(spu
->irq_c0
, sizeof (spu
->irq_c0
), "spe%02d.0",
423 ret
= request_irq(spu
->irqs
[0], spu_irq_class_0
,
429 if (spu
->irqs
[1] != NO_IRQ
) {
430 snprintf(spu
->irq_c1
, sizeof (spu
->irq_c1
), "spe%02d.1",
432 ret
= request_irq(spu
->irqs
[1], spu_irq_class_1
,
438 if (spu
->irqs
[2] != NO_IRQ
) {
439 snprintf(spu
->irq_c2
, sizeof (spu
->irq_c2
), "spe%02d.2",
441 ret
= request_irq(spu
->irqs
[2], spu_irq_class_2
,
450 if (spu
->irqs
[1] != NO_IRQ
)
451 free_irq(spu
->irqs
[1], spu
);
453 if (spu
->irqs
[0] != NO_IRQ
)
454 free_irq(spu
->irqs
[0], spu
);
459 static void spu_free_irqs(struct spu
*spu
)
461 if (spu
->irqs
[0] != NO_IRQ
)
462 free_irq(spu
->irqs
[0], spu
);
463 if (spu
->irqs
[1] != NO_IRQ
)
464 free_irq(spu
->irqs
[1], spu
);
465 if (spu
->irqs
[2] != NO_IRQ
)
466 free_irq(spu
->irqs
[2], spu
);
469 void spu_init_channels(struct spu
*spu
)
471 static const struct {
475 { 0x00, 1, }, { 0x01, 1, }, { 0x03, 1, }, { 0x04, 1, },
476 { 0x18, 1, }, { 0x19, 1, }, { 0x1b, 1, }, { 0x1d, 1, },
478 { 0x00, 0, }, { 0x03, 0, }, { 0x04, 0, }, { 0x15, 16, },
479 { 0x17, 1, }, { 0x18, 0, }, { 0x19, 0, }, { 0x1b, 0, },
480 { 0x1c, 1, }, { 0x1d, 0, }, { 0x1e, 1, },
482 struct spu_priv2 __iomem
*priv2
;
487 /* initialize all channel data to zero */
488 for (i
= 0; i
< ARRAY_SIZE(zero_list
); i
++) {
491 out_be64(&priv2
->spu_chnlcntptr_RW
, zero_list
[i
].channel
);
492 for (count
= 0; count
< zero_list
[i
].count
; count
++)
493 out_be64(&priv2
->spu_chnldata_RW
, 0);
496 /* initialize channel counts to meaningful values */
497 for (i
= 0; i
< ARRAY_SIZE(count_list
); i
++) {
498 out_be64(&priv2
->spu_chnlcntptr_RW
, count_list
[i
].channel
);
499 out_be64(&priv2
->spu_chnlcnt_RW
, count_list
[i
].count
);
502 EXPORT_SYMBOL_GPL(spu_init_channels
);
504 static int spu_shutdown(struct sys_device
*sysdev
)
506 struct spu
*spu
= container_of(sysdev
, struct spu
, sysdev
);
509 spu_destroy_spu(spu
);
513 static struct sysdev_class spu_sysdev_class
= {
515 .shutdown
= spu_shutdown
,
518 int spu_add_sysdev_attr(struct sysdev_attribute
*attr
)
522 mutex_lock(&spu_full_list_mutex
);
523 list_for_each_entry(spu
, &spu_full_list
, full_list
)
524 sysdev_create_file(&spu
->sysdev
, attr
);
525 mutex_unlock(&spu_full_list_mutex
);
529 EXPORT_SYMBOL_GPL(spu_add_sysdev_attr
);
531 int spu_add_sysdev_attr_group(struct attribute_group
*attrs
)
536 mutex_lock(&spu_full_list_mutex
);
537 list_for_each_entry(spu
, &spu_full_list
, full_list
) {
538 rc
= sysfs_create_group(&spu
->sysdev
.kobj
, attrs
);
540 /* we're in trouble here, but try unwinding anyway */
542 printk(KERN_ERR
"%s: can't create sysfs group '%s'\n",
543 __func__
, attrs
->name
);
545 list_for_each_entry_continue_reverse(spu
,
546 &spu_full_list
, full_list
)
547 sysfs_remove_group(&spu
->sysdev
.kobj
, attrs
);
552 mutex_unlock(&spu_full_list_mutex
);
556 EXPORT_SYMBOL_GPL(spu_add_sysdev_attr_group
);
559 void spu_remove_sysdev_attr(struct sysdev_attribute
*attr
)
563 mutex_lock(&spu_full_list_mutex
);
564 list_for_each_entry(spu
, &spu_full_list
, full_list
)
565 sysdev_remove_file(&spu
->sysdev
, attr
);
566 mutex_unlock(&spu_full_list_mutex
);
568 EXPORT_SYMBOL_GPL(spu_remove_sysdev_attr
);
570 void spu_remove_sysdev_attr_group(struct attribute_group
*attrs
)
574 mutex_lock(&spu_full_list_mutex
);
575 list_for_each_entry(spu
, &spu_full_list
, full_list
)
576 sysfs_remove_group(&spu
->sysdev
.kobj
, attrs
);
577 mutex_unlock(&spu_full_list_mutex
);
579 EXPORT_SYMBOL_GPL(spu_remove_sysdev_attr_group
);
581 static int spu_create_sysdev(struct spu
*spu
)
585 spu
->sysdev
.id
= spu
->number
;
586 spu
->sysdev
.cls
= &spu_sysdev_class
;
587 ret
= sysdev_register(&spu
->sysdev
);
589 printk(KERN_ERR
"Can't register SPU %d with sysfs\n",
594 sysfs_add_device_to_node(&spu
->sysdev
, spu
->node
);
599 static int __init
create_spu(void *data
)
608 spu
= kzalloc(sizeof (*spu
), GFP_KERNEL
);
612 spu
->alloc_state
= SPU_FREE
;
614 spin_lock_init(&spu
->register_lock
);
615 spin_lock(&spu_lock
);
616 spu
->number
= number
++;
617 spin_unlock(&spu_lock
);
619 ret
= spu_create_spu(spu
, data
);
624 spu_mfc_sdr_setup(spu
);
625 spu_mfc_sr1_set(spu
, 0x33);
626 ret
= spu_request_irqs(spu
);
630 ret
= spu_create_sysdev(spu
);
634 mutex_lock(&cbe_spu_info
[spu
->node
].list_mutex
);
635 list_add(&spu
->cbe_list
, &cbe_spu_info
[spu
->node
].spus
);
636 cbe_spu_info
[spu
->node
].n_spus
++;
637 mutex_unlock(&cbe_spu_info
[spu
->node
].list_mutex
);
639 mutex_lock(&spu_full_list_mutex
);
640 spin_lock_irqsave(&spu_full_list_lock
, flags
);
641 list_add(&spu
->full_list
, &spu_full_list
);
642 spin_unlock_irqrestore(&spu_full_list_lock
, flags
);
643 mutex_unlock(&spu_full_list_mutex
);
645 spu
->stats
.util_state
= SPU_UTIL_IDLE_LOADED
;
647 spu
->stats
.tstamp
= timespec_to_ns(&ts
);
649 INIT_LIST_HEAD(&spu
->aff_list
);
656 spu_destroy_spu(spu
);
663 static const char *spu_state_names
[] = {
664 "user", "system", "iowait", "idle"
667 static unsigned long long spu_acct_time(struct spu
*spu
,
668 enum spu_utilization_state state
)
671 unsigned long long time
= spu
->stats
.times
[state
];
674 * If the spu is idle or the context is stopped, utilization
675 * statistics are not updated. Apply the time delta from the
676 * last recorded state of the spu.
678 if (spu
->stats
.util_state
== state
) {
680 time
+= timespec_to_ns(&ts
) - spu
->stats
.tstamp
;
683 return time
/ NSEC_PER_MSEC
;
687 static ssize_t
spu_stat_show(struct sys_device
*sysdev
, char *buf
)
689 struct spu
*spu
= container_of(sysdev
, struct spu
, sysdev
);
691 return sprintf(buf
, "%s %llu %llu %llu %llu "
692 "%llu %llu %llu %llu %llu %llu %llu %llu\n",
693 spu_state_names
[spu
->stats
.util_state
],
694 spu_acct_time(spu
, SPU_UTIL_USER
),
695 spu_acct_time(spu
, SPU_UTIL_SYSTEM
),
696 spu_acct_time(spu
, SPU_UTIL_IOWAIT
),
697 spu_acct_time(spu
, SPU_UTIL_IDLE_LOADED
),
698 spu
->stats
.vol_ctx_switch
,
699 spu
->stats
.invol_ctx_switch
,
704 spu
->stats
.class2_intr
,
705 spu
->stats
.libassist
);
708 static SYSDEV_ATTR(stat
, 0644, spu_stat_show
, NULL
);
710 static int __init
init_spu_base(void)
714 for (i
= 0; i
< MAX_NUMNODES
; i
++) {
715 mutex_init(&cbe_spu_info
[i
].list_mutex
);
716 INIT_LIST_HEAD(&cbe_spu_info
[i
].spus
);
719 if (!spu_management_ops
)
722 /* create sysdev class for spus */
723 ret
= sysdev_class_register(&spu_sysdev_class
);
727 ret
= spu_enumerate_spus(create_spu
);
730 printk(KERN_WARNING
"%s: Error initializing spus\n",
732 goto out_unregister_sysdev_class
;
737 * We cannot put the forward declaration in
738 * <linux/linux_logo.h> because of conflicting session type
739 * conflicts for const and __initdata with different compiler
742 extern const struct linux_logo logo_spe_clut224
;
744 fb_append_extra_logo(&logo_spe_clut224
, ret
);
747 mutex_lock(&spu_full_list_mutex
);
748 xmon_register_spus(&spu_full_list
);
749 crash_register_spus(&spu_full_list
);
750 mutex_unlock(&spu_full_list_mutex
);
751 spu_add_sysdev_attr(&attr_stat
);
757 out_unregister_sysdev_class
:
758 sysdev_class_unregister(&spu_sysdev_class
);
762 module_init(init_spu_base
);
764 MODULE_LICENSE("GPL");
765 MODULE_AUTHOR("Arnd Bergmann <arndb@de.ibm.com>");