2 * Cell Broadband Engine OProfile Support
4 * (C) Copyright IBM Corporation 2006
6 * Author: Maynard Johnson <maynardj@us.ibm.com>
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
14 /* The purpose of this file is to handle SPU event task switching
15 * and to record SPU context information into the OProfile
18 * Additionally, the spu_sync_buffer function is provided as a helper
19 * for recoding actual SPU program counter samples to the event buffer.
21 #include <linux/dcookies.h>
22 #include <linux/kref.h>
25 #include <linux/module.h>
26 #include <linux/notifier.h>
27 #include <linux/numa.h>
28 #include <linux/oprofile.h>
29 #include <linux/slab.h>
30 #include <linux/spinlock.h>
33 #define RELEASE_ALL 9999
35 static DEFINE_SPINLOCK(buffer_lock
);
36 static DEFINE_SPINLOCK(cache_lock
);
37 static int num_spu_nodes
;
38 int spu_prof_num_nodes
;
40 struct spu_buffer spu_buff
[MAX_NUMNODES
* SPUS_PER_NODE
];
41 struct delayed_work spu_work
;
42 static unsigned max_spu_buff
;
44 static void spu_buff_add(unsigned long int value
, int spu
)
46 /* spu buff is a circular buffer. Add entries to the
47 * head. Head is the index to store the next value.
48 * The buffer is full when there is one available entry
49 * in the queue, i.e. head and tail can't be equal.
50 * That way we can tell the difference between the
51 * buffer being full versus empty.
53 * ASSUPTION: the buffer_lock is held when this function
54 * is called to lock the buffer, head and tail.
58 if (spu_buff
[spu
].head
>= spu_buff
[spu
].tail
) {
59 if ((spu_buff
[spu
].head
- spu_buff
[spu
].tail
)
63 } else if (spu_buff
[spu
].tail
> spu_buff
[spu
].head
) {
64 if ((spu_buff
[spu
].tail
- spu_buff
[spu
].head
)
70 spu_buff
[spu
].buff
[spu_buff
[spu
].head
] = value
;
73 if (spu_buff
[spu
].head
>= max_spu_buff
)
74 spu_buff
[spu
].head
= 0;
76 /* From the user's perspective make the SPU buffer
77 * size management/overflow look like we are using
78 * per cpu buffers. The user uses the same
79 * per cpu parameter to adjust the SPU buffer size.
80 * Increment the sample_lost_overflow to inform
81 * the user the buffer size needs to be increased.
83 oprofile_cpu_buffer_inc_smpl_lost();
87 /* This function copies the per SPU buffers to the
88 * OProfile kernel buffer.
90 void sync_spu_buff(void)
96 for (spu
= 0; spu
< num_spu_nodes
; spu
++) {
97 /* In case there was an issue and the buffer didn't
98 * get created skip it.
100 if (spu_buff
[spu
].buff
== NULL
)
103 /* Hold the lock to make sure the head/tail
104 * doesn't change while spu_buff_add() is
105 * deciding if the buffer is full or not.
106 * Being a little paranoid.
108 spin_lock_irqsave(&buffer_lock
, flags
);
109 curr_head
= spu_buff
[spu
].head
;
110 spin_unlock_irqrestore(&buffer_lock
, flags
);
112 /* Transfer the current contents to the kernel buffer.
113 * data can still be added to the head of the buffer.
115 oprofile_put_buff(spu_buff
[spu
].buff
,
117 curr_head
, max_spu_buff
);
119 spin_lock_irqsave(&buffer_lock
, flags
);
120 spu_buff
[spu
].tail
= curr_head
;
121 spin_unlock_irqrestore(&buffer_lock
, flags
);
126 static void wq_sync_spu_buff(struct work_struct
*work
)
128 /* move data from spu buffers to kernel buffer */
131 /* only reschedule if profiling is not done */
132 if (spu_prof_running
)
133 schedule_delayed_work(&spu_work
, DEFAULT_TIMER_EXPIRE
);
136 /* Container for caching information about an active SPU task. */
138 struct vma_to_fileoffset_map
*map
;
139 struct spu
*the_spu
; /* needed to access pointer to local_store */
140 struct kref cache_ref
;
143 static struct cached_info
*spu_info
[MAX_NUMNODES
* 8];
145 static void destroy_cached_info(struct kref
*kref
)
147 struct cached_info
*info
;
149 info
= container_of(kref
, struct cached_info
, cache_ref
);
150 vma_map_free(info
->map
);
152 module_put(THIS_MODULE
);
155 /* Return the cached_info for the passed SPU number.
156 * ATTENTION: Callers are responsible for obtaining the
157 * cache_lock if needed prior to invoking this function.
159 static struct cached_info
*get_cached_info(struct spu
*the_spu
, int spu_num
)
162 struct cached_info
*ret_info
;
164 if (spu_num
>= num_spu_nodes
) {
165 printk(KERN_ERR
"SPU_PROF: "
166 "%s, line %d: Invalid index %d into spu info cache\n",
167 __func__
, __LINE__
, spu_num
);
171 if (!spu_info
[spu_num
] && the_spu
) {
172 ref
= spu_get_profile_private_kref(the_spu
->ctx
);
174 spu_info
[spu_num
] = container_of(ref
, struct cached_info
, cache_ref
);
175 kref_get(&spu_info
[spu_num
]->cache_ref
);
179 ret_info
= spu_info
[spu_num
];
185 /* Looks for cached info for the passed spu. If not found, the
186 * cached info is created for the passed spu.
187 * Returns 0 for success; otherwise, -1 for error.
190 prepare_cached_spu_info(struct spu
*spu
, unsigned long objectId
)
193 struct vma_to_fileoffset_map
*new_map
;
195 struct cached_info
*info
;
197 /* We won't bother getting cache_lock here since
198 * don't do anything with the cached_info that's returned.
200 info
= get_cached_info(spu
, spu
->number
);
203 pr_debug("Found cached SPU info.\n");
207 /* Create cached_info and set spu_info[spu->number] to point to it.
208 * spu->number is a system-wide value, not a per-node value.
210 info
= kzalloc(sizeof(struct cached_info
), GFP_KERNEL
);
212 printk(KERN_ERR
"SPU_PROF: "
213 "%s, line %d: create vma_map failed\n",
218 new_map
= create_vma_map(spu
, objectId
);
220 printk(KERN_ERR
"SPU_PROF: "
221 "%s, line %d: create vma_map failed\n",
227 pr_debug("Created vma_map\n");
230 kref_init(&info
->cache_ref
);
231 spin_lock_irqsave(&cache_lock
, flags
);
232 spu_info
[spu
->number
] = info
;
233 /* Increment count before passing off ref to SPUFS. */
234 kref_get(&info
->cache_ref
);
236 /* We increment the module refcount here since SPUFS is
237 * responsible for the final destruction of the cached_info,
238 * and it must be able to access the destroy_cached_info()
239 * function defined in the OProfile module. We decrement
240 * the module refcount in destroy_cached_info.
242 try_module_get(THIS_MODULE
);
243 spu_set_profile_private_kref(spu
->ctx
, &info
->cache_ref
,
244 destroy_cached_info
);
245 spin_unlock_irqrestore(&cache_lock
, flags
);
255 * NOTE: The caller is responsible for locking the
256 * cache_lock prior to calling this function.
258 static int release_cached_info(int spu_index
)
262 if (spu_index
== RELEASE_ALL
) {
266 if (spu_index
>= num_spu_nodes
) {
267 printk(KERN_ERR
"SPU_PROF: "
269 "Invalid index %d into spu info cache\n",
270 __func__
, __LINE__
, spu_index
);
276 for (; index
< end
; index
++) {
277 if (spu_info
[index
]) {
278 kref_put(&spu_info
[index
]->cache_ref
,
279 destroy_cached_info
);
280 spu_info
[index
] = NULL
;
288 /* The source code for fast_get_dcookie was "borrowed"
289 * from drivers/oprofile/buffer_sync.c.
292 /* Optimisation. We can manage without taking the dcookie sem
293 * because we cannot reach this code without at least one
294 * dcookie user still being registered (namely, the reader
295 * of the event buffer).
297 static inline unsigned long fast_get_dcookie(struct path
*path
)
299 unsigned long cookie
;
301 if (path
->dentry
->d_flags
& DCACHE_COOKIE
)
302 return (unsigned long)path
->dentry
;
303 get_dcookie(path
, &cookie
);
307 /* Look up the dcookie for the task's mm->exe_file,
308 * which corresponds loosely to "application name". Also, determine
309 * the offset for the SPU ELF object. If computed offset is
310 * non-zero, it implies an embedded SPU object; otherwise, it's a
311 * separate SPU binary, in which case we retrieve it's dcookie.
312 * For the embedded case, we must determine if SPU ELF is embedded
313 * in the executable application or another file (i.e., shared lib).
314 * If embedded in a shared lib, we must get the dcookie and return
315 * that to the caller.
318 get_exec_dcookie_and_offset(struct spu
*spu
, unsigned int *offsetp
,
319 unsigned long *spu_bin_dcookie
,
320 unsigned long spu_ref
)
322 unsigned long app_cookie
= 0;
323 unsigned int my_offset
= 0;
324 struct vm_area_struct
*vma
;
325 struct mm_struct
*mm
= spu
->mm
;
330 down_read(&mm
->mmap_sem
);
333 app_cookie
= fast_get_dcookie(&mm
->exe_file
->f_path
);
334 pr_debug("got dcookie for %s\n",
335 mm
->exe_file
->f_dentry
->d_name
.name
);
338 for (vma
= mm
->mmap
; vma
; vma
= vma
->vm_next
) {
339 if (vma
->vm_start
> spu_ref
|| vma
->vm_end
<= spu_ref
)
341 my_offset
= spu_ref
- vma
->vm_start
;
343 goto fail_no_image_cookie
;
345 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n",
347 vma
->vm_file
->f_dentry
->d_name
.name
);
348 *offsetp
= my_offset
;
352 *spu_bin_dcookie
= fast_get_dcookie(&vma
->vm_file
->f_path
);
353 pr_debug("got dcookie for %s\n", vma
->vm_file
->f_dentry
->d_name
.name
);
355 up_read(&mm
->mmap_sem
);
360 fail_no_image_cookie
:
361 up_read(&mm
->mmap_sem
);
363 printk(KERN_ERR
"SPU_PROF: "
364 "%s, line %d: Cannot find dcookie for SPU binary\n",
371 /* This function finds or creates cached context information for the
372 * passed SPU and records SPU context information into the OProfile
375 static int process_context_switch(struct spu
*spu
, unsigned long objectId
)
379 unsigned int offset
= 0;
380 unsigned long spu_cookie
= 0, app_dcookie
;
382 retval
= prepare_cached_spu_info(spu
, objectId
);
386 /* Get dcookie first because a mutex_lock is taken in that
387 * code path, so interrupts must not be disabled.
389 app_dcookie
= get_exec_dcookie_and_offset(spu
, &offset
, &spu_cookie
, objectId
);
390 if (!app_dcookie
|| !spu_cookie
) {
395 /* Record context info in event buffer */
396 spin_lock_irqsave(&buffer_lock
, flags
);
397 spu_buff_add(ESCAPE_CODE
, spu
->number
);
398 spu_buff_add(SPU_CTX_SWITCH_CODE
, spu
->number
);
399 spu_buff_add(spu
->number
, spu
->number
);
400 spu_buff_add(spu
->pid
, spu
->number
);
401 spu_buff_add(spu
->tgid
, spu
->number
);
402 spu_buff_add(app_dcookie
, spu
->number
);
403 spu_buff_add(spu_cookie
, spu
->number
);
404 spu_buff_add(offset
, spu
->number
);
406 /* Set flag to indicate SPU PC data can now be written out. If
407 * the SPU program counter data is seen before an SPU context
408 * record is seen, the postprocessing will fail.
410 spu_buff
[spu
->number
].ctx_sw_seen
= 1;
412 spin_unlock_irqrestore(&buffer_lock
, flags
);
413 smp_wmb(); /* insure spu event buffer updates are written */
414 /* don't want entries intermingled... */
420 * This function is invoked on either a bind_context or unbind_context.
421 * If called for an unbind_context, the val arg is 0; otherwise,
422 * it is the object-id value for the spu context.
423 * The data arg is of type 'struct spu *'.
425 static int spu_active_notify(struct notifier_block
*self
, unsigned long val
,
430 struct spu
*the_spu
= data
;
432 pr_debug("SPU event notification arrived\n");
434 spin_lock_irqsave(&cache_lock
, flags
);
435 retval
= release_cached_info(the_spu
->number
);
436 spin_unlock_irqrestore(&cache_lock
, flags
);
438 retval
= process_context_switch(the_spu
, val
);
443 static struct notifier_block spu_active
= {
444 .notifier_call
= spu_active_notify
,
447 static int number_of_online_nodes(void)
451 for_each_online_cpu(cpu
) {
452 tmp
= cbe_cpu_to_node(cpu
) + 1;
459 static int oprofile_spu_buff_create(void)
463 max_spu_buff
= oprofile_get_cpu_buffer_size();
465 for (spu
= 0; spu
< num_spu_nodes
; spu
++) {
466 /* create circular buffers to store the data in.
467 * use locks to manage accessing the buffers
469 spu_buff
[spu
].head
= 0;
470 spu_buff
[spu
].tail
= 0;
473 * Create a buffer for each SPU. Can't reliably
474 * create a single buffer for all spus due to not
475 * enough contiguous kernel memory.
478 spu_buff
[spu
].buff
= kzalloc((max_spu_buff
479 * sizeof(unsigned long)),
482 if (!spu_buff
[spu
].buff
) {
483 printk(KERN_ERR
"SPU_PROF: "
484 "%s, line %d: oprofile_spu_buff_create "
485 "failed to allocate spu buffer %d.\n",
486 __func__
, __LINE__
, spu
);
488 /* release the spu buffers that have been allocated */
490 kfree(spu_buff
[spu
].buff
);
491 spu_buff
[spu
].buff
= 0;
500 /* The main purpose of this function is to synchronize
501 * OProfile with SPUFS by registering to be notified of
504 * NOTE: When profiling SPUs, we must ensure that only
505 * spu_sync_start is invoked and not the generic sync_start
506 * in drivers/oprofile/oprof.c. A return value of
507 * SKIP_GENERIC_SYNC or SYNC_START_ERROR will
510 int spu_sync_start(void)
513 int ret
= SKIP_GENERIC_SYNC
;
515 unsigned long flags
= 0;
517 spu_prof_num_nodes
= number_of_online_nodes();
518 num_spu_nodes
= spu_prof_num_nodes
* 8;
519 INIT_DELAYED_WORK(&spu_work
, wq_sync_spu_buff
);
521 /* create buffer for storing the SPU data to put in
524 ret
= oprofile_spu_buff_create();
528 spin_lock_irqsave(&buffer_lock
, flags
);
529 for (spu
= 0; spu
< num_spu_nodes
; spu
++) {
530 spu_buff_add(ESCAPE_CODE
, spu
);
531 spu_buff_add(SPU_PROFILING_CODE
, spu
);
532 spu_buff_add(num_spu_nodes
, spu
);
534 spin_unlock_irqrestore(&buffer_lock
, flags
);
536 for (spu
= 0; spu
< num_spu_nodes
; spu
++) {
537 spu_buff
[spu
].ctx_sw_seen
= 0;
538 spu_buff
[spu
].last_guard_val
= 0;
541 /* Register for SPU events */
542 register_ret
= spu_switch_event_register(&spu_active
);
544 ret
= SYNC_START_ERROR
;
548 pr_debug("spu_sync_start -- running.\n");
553 /* Record SPU program counter samples to the oprofile event buffer. */
554 void spu_sync_buffer(int spu_num
, unsigned int *samples
,
557 unsigned long long file_offset
;
560 struct vma_to_fileoffset_map
*map
;
562 unsigned long long spu_num_ll
= spu_num
;
563 unsigned long long spu_num_shifted
= spu_num_ll
<< 32;
564 struct cached_info
*c_info
;
566 /* We need to obtain the cache_lock here because it's
567 * possible that after getting the cached_info, the SPU job
568 * corresponding to this cached_info may end, thus resulting
569 * in the destruction of the cached_info.
571 spin_lock_irqsave(&cache_lock
, flags
);
572 c_info
= get_cached_info(NULL
, spu_num
);
574 /* This legitimately happens when the SPU task ends before all
575 * samples are recorded.
576 * No big deal -- so we just drop a few samples.
578 pr_debug("SPU_PROF: No cached SPU contex "
579 "for SPU #%d. Dropping samples.\n", spu_num
);
584 the_spu
= c_info
->the_spu
;
585 spin_lock(&buffer_lock
);
586 for (i
= 0; i
< num_samples
; i
++) {
587 unsigned int sample
= *(samples
+i
);
592 file_offset
= vma_map_lookup( map
, sample
, the_spu
, &grd_val
);
594 /* If overlays are used by this SPU application, the guard
595 * value is non-zero, indicating which overlay section is in
596 * use. We need to discard samples taken during the time
597 * period which an overlay occurs (i.e., guard value changes).
599 if (grd_val
&& grd_val
!= spu_buff
[spu_num
].last_guard_val
) {
600 spu_buff
[spu_num
].last_guard_val
= grd_val
;
601 /* Drop the rest of the samples. */
605 /* We must ensure that the SPU context switch has been written
606 * out before samples for the SPU. Otherwise, the SPU context
607 * information is not available and the postprocessing of the
608 * SPU PC will fail with no available anonymous map information.
610 if (spu_buff
[spu_num
].ctx_sw_seen
)
611 spu_buff_add((file_offset
| spu_num_shifted
),
614 spin_unlock(&buffer_lock
);
616 spin_unlock_irqrestore(&cache_lock
, flags
);
620 int spu_sync_stop(void)
622 unsigned long flags
= 0;
626 ret
= spu_switch_event_unregister(&spu_active
);
629 printk(KERN_ERR
"SPU_PROF: "
630 "%s, line %d: spu_switch_event_unregister " \
632 __func__
, __LINE__
, ret
);
634 /* flush any remaining data in the per SPU buffers */
637 spin_lock_irqsave(&cache_lock
, flags
);
638 ret
= release_cached_info(RELEASE_ALL
);
639 spin_unlock_irqrestore(&cache_lock
, flags
);
641 /* remove scheduled work queue item rather then waiting
642 * for every queued entry to execute. Then flush pending
643 * system wide buffer to event buffer.
645 cancel_delayed_work(&spu_work
);
647 for (k
= 0; k
< num_spu_nodes
; k
++) {
648 spu_buff
[k
].ctx_sw_seen
= 0;
651 * spu_sys_buff will be null if there was a problem
652 * allocating the buffer. Only delete if it exists.
654 kfree(spu_buff
[k
].buff
);
655 spu_buff
[k
].buff
= 0;
657 pr_debug("spu_sync_stop -- done.\n");