4 * This provides a low-level interface to the hardware's Debug Store
5 * feature that is used for branch trace store (BTS) and
6 * precise-event based sampling (PEBS).
9 * - per-thread and per-cpu allocation of BTS and PEBS
10 * - buffer memory allocation (optional)
11 * - buffer overflow handling
15 * - get_task_struct on all parameter tasks
16 * - current is allowed to trace parameter tasks
19 * Copyright (C) 2007-2008 Intel Corporation.
20 * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008
28 #include <linux/errno.h>
29 #include <linux/string.h>
30 #include <linux/slab.h>
31 #include <linux/sched.h>
36 * The configuration for a particular DS hardware implementation.
38 struct ds_configuration
{
39 /* the size of the DS structure in bytes */
40 unsigned char sizeof_ds
;
41 /* the size of one pointer-typed field in the DS structure in bytes;
42 this covers the first 8 fields related to buffer management. */
43 unsigned char sizeof_field
;
44 /* the size of a BTS/PEBS record in bytes */
45 unsigned char sizeof_rec
[2];
47 static struct ds_configuration ds_cfg
;
51 * Debug Store (DS) save area configuration (see Intel64 and IA32
52 * Architectures Software Developer's Manual, section 18.5)
54 * The DS configuration consists of the following fields; different
55 * architetures vary in the size of those fields.
56 * - double-word aligned base linear address of the BTS buffer
57 * - write pointer into the BTS buffer
58 * - end linear address of the BTS buffer (one byte beyond the end of
60 * - interrupt pointer into BTS buffer
61 * (interrupt occurs when write pointer passes interrupt pointer)
62 * - double-word aligned base linear address of the PEBS buffer
63 * - write pointer into the PEBS buffer
64 * - end linear address of the PEBS buffer (one byte beyond the end of
66 * - interrupt pointer into PEBS buffer
67 * (interrupt occurs when write pointer passes interrupt pointer)
68 * - value to which counter is reset following counter overflow
70 * Later architectures use 64bit pointers throughout, whereas earlier
71 * architectures use 32bit pointers in 32bit mode.
74 * We compute the base address for the first 8 fields based on:
75 * - the field size stored in the DS configuration
76 * - the relative field position
77 * - an offset giving the start of the respective region
79 * This offset is further used to index various arrays holding
80 * information for BTS and PEBS at the respective index.
82 * On later 32bit processors, we only access the lower 32bit of the
83 * 64bit pointer fields. The upper halves will be zeroed out.
90 ds_interrupt_threshold
,
98 static inline unsigned long ds_get(const unsigned char *base
,
99 enum ds_qualifier qual
, enum ds_field field
)
101 base
+= (ds_cfg
.sizeof_field
* (field
+ (4 * qual
)));
102 return *(unsigned long *)base
;
105 static inline void ds_set(unsigned char *base
, enum ds_qualifier qual
,
106 enum ds_field field
, unsigned long value
)
108 base
+= (ds_cfg
.sizeof_field
* (field
+ (4 * qual
)));
109 (*(unsigned long *)base
) = value
;
114 * Locking is done only for allocating BTS or PEBS resources and for
115 * guarding context and buffer memory allocation.
117 * Most functions require the current task to own the ds context part
118 * they are going to access. All the locking is done when validating
119 * access to the context.
121 static spinlock_t ds_lock
= __SPIN_LOCK_UNLOCKED(ds_lock
);
124 * Validate that the current task is allowed to access the BTS/PEBS
125 * buffer of the parameter task.
127 * Returns 0, if access is granted; -Eerrno, otherwise.
129 static inline int ds_validate_access(struct ds_context
*context
,
130 enum ds_qualifier qual
)
135 if (context
->owner
[qual
] == current
)
143 * We either support (system-wide) per-cpu or per-thread allocation.
144 * We distinguish the two based on the task_struct pointer, where a
145 * NULL pointer indicates per-cpu allocation for the current cpu.
147 * Allocations are use-counted. As soon as resources are allocated,
148 * further allocations must be of the same type (per-cpu or
149 * per-thread). We model this by counting allocations (i.e. the number
150 * of tracers of a certain type) for one type negatively:
152 * >0 number of per-thread tracers
153 * <0 number of per-cpu tracers
155 * The below functions to get and put tracers and to check the
156 * allocation type require the ds_lock to be held by the caller.
158 * Tracers essentially gives the number of ds contexts for a certain
159 * type of allocation.
163 static inline void get_tracer(struct task_struct
*task
)
165 tracers
+= (task
? 1 : -1);
168 static inline void put_tracer(struct task_struct
*task
)
170 tracers
-= (task
? 1 : -1);
173 static inline int check_tracer(struct task_struct
*task
)
175 return (task
? (tracers
>= 0) : (tracers
<= 0));
180 * The DS context is either attached to a thread or to a cpu:
181 * - in the former case, the thread_struct contains a pointer to the
183 * - in the latter case, we use a static array of per-cpu context
186 * Contexts are use-counted. They are allocated on first access and
187 * deallocated when the last user puts the context.
189 * We distinguish between an allocating and a non-allocating get of a
191 * - the allocating get is used for requesting BTS/PEBS resources. It
192 * requires the caller to hold the global ds_lock.
193 * - the non-allocating get is used for all other cases. A
194 * non-existing context indicates an error. It acquires and releases
195 * the ds_lock itself for obtaining the context.
197 * A context and its DS configuration are allocated and deallocated
198 * together. A context always has a DS configuration of the
201 static DEFINE_PER_CPU(struct ds_context
*, system_context
);
203 #define this_system_context per_cpu(system_context, smp_processor_id())
206 * Returns the pointer to the parameter task's context or to the
207 * system-wide context, if task is NULL.
209 * Increases the use count of the returned context, if not NULL.
211 static inline struct ds_context
*ds_get_context(struct task_struct
*task
)
213 struct ds_context
*context
;
217 context
= (task
? task
->thread
.ds_ctx
: this_system_context
);
221 spin_unlock(&ds_lock
);
227 * Same as ds_get_context, but allocates the context and it's DS
228 * structure, if necessary; returns NULL; if out of memory.
230 * pre: requires ds_lock to be held
232 static inline struct ds_context
*ds_alloc_context(struct task_struct
*task
)
234 struct ds_context
**p_context
=
235 (task
? &task
->thread
.ds_ctx
: &this_system_context
);
236 struct ds_context
*context
= *p_context
;
239 spin_unlock(&ds_lock
);
241 context
= kzalloc(sizeof(*context
), GFP_KERNEL
);
248 context
->ds
= kzalloc(ds_cfg
.sizeof_ds
, GFP_KERNEL
);
257 * Check for race - another CPU could have allocated
266 *p_context
= context
;
268 context
->this = p_context
;
269 context
->task
= task
;
272 set_tsk_thread_flag(task
, TIF_DS_AREA_MSR
);
274 if (!task
|| (task
== current
))
275 wrmsr(MSR_IA32_DS_AREA
, (unsigned long)context
->ds
, 0);
286 * Decreases the use count of the parameter context, if not NULL.
287 * Deallocates the context, if the use count reaches zero.
289 static inline void ds_put_context(struct ds_context
*context
)
296 if (--context
->count
)
299 *(context
->this) = NULL
;
302 clear_tsk_thread_flag(context
->task
, TIF_DS_AREA_MSR
);
304 if (!context
->task
|| (context
->task
== current
))
305 wrmsrl(MSR_IA32_DS_AREA
, 0);
307 put_tracer(context
->task
);
309 /* free any leftover buffers from tracers that did not
310 * deallocate them properly. */
311 kfree(context
->buffer
[ds_bts
]);
312 kfree(context
->buffer
[ds_pebs
]);
316 spin_unlock(&ds_lock
);
321 * Handle a buffer overflow
323 * task: the task whose buffers are overflowing;
324 * NULL for a buffer overflow on the current cpu
325 * context: the ds context
326 * qual: the buffer type
328 static void ds_overflow(struct task_struct
*task
, struct ds_context
*context
,
329 enum ds_qualifier qual
)
334 if (context
->callback
[qual
])
335 (*context
->callback
[qual
])(task
);
337 /* todo: do some more overflow handling */
342 * Allocate a non-pageable buffer of the parameter size.
343 * Checks the memory and the locked memory rlimit.
345 * Returns the buffer, if successful;
346 * NULL, if out of memory or rlimit exceeded.
348 * size: the requested buffer size in bytes
349 * pages (out): if not NULL, contains the number of pages reserved
351 static inline void *ds_allocate_buffer(size_t size
, unsigned int *pages
)
353 unsigned long rlim
, vm
, pgsz
;
356 pgsz
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
358 rlim
= current
->signal
->rlim
[RLIMIT_AS
].rlim_cur
>> PAGE_SHIFT
;
359 vm
= current
->mm
->total_vm
+ pgsz
;
363 rlim
= current
->signal
->rlim
[RLIMIT_MEMLOCK
].rlim_cur
>> PAGE_SHIFT
;
364 vm
= current
->mm
->locked_vm
+ pgsz
;
368 buffer
= kzalloc(size
, GFP_KERNEL
);
372 current
->mm
->total_vm
+= pgsz
;
373 current
->mm
->locked_vm
+= pgsz
;
381 static int ds_request(struct task_struct
*task
, void *base
, size_t size
,
382 ds_ovfl_callback_t ovfl
, enum ds_qualifier qual
)
384 struct ds_context
*context
;
385 unsigned long buffer
, adj
;
386 const unsigned long alignment
= (1 << 3);
389 if (!ds_cfg
.sizeof_ds
)
392 /* we require some space to do alignment adjustments below */
393 if (size
< (alignment
+ ds_cfg
.sizeof_rec
[qual
]))
396 /* buffer overflow notification is not yet implemented */
404 context
= ds_alloc_context(task
);
409 if (!check_tracer(task
))
413 if (context
->owner
[qual
] == current
)
416 if (context
->owner
[qual
] != NULL
)
418 context
->owner
[qual
] = current
;
420 spin_unlock(&ds_lock
);
425 base
= ds_allocate_buffer(size
, &context
->pages
[qual
]);
429 context
->buffer
[qual
] = base
;
433 context
->callback
[qual
] = ovfl
;
435 /* adjust the buffer address and size to meet alignment
437 * - buffer is double-word aligned
438 * - size is multiple of record size
440 * We checked the size at the very beginning; we have enough
441 * space to do the adjustment.
443 buffer
= (unsigned long)base
;
445 adj
= ALIGN(buffer
, alignment
) - buffer
;
449 size
/= ds_cfg
.sizeof_rec
[qual
];
450 size
*= ds_cfg
.sizeof_rec
[qual
];
452 ds_set(context
->ds
, qual
, ds_buffer_base
, buffer
);
453 ds_set(context
->ds
, qual
, ds_index
, buffer
);
454 ds_set(context
->ds
, qual
, ds_absolute_maximum
, buffer
+ size
);
457 /* todo: select a suitable interrupt threshold */
459 ds_set(context
->ds
, qual
,
460 ds_interrupt_threshold
, buffer
+ size
+ 1);
462 /* we keep the context until ds_release */
466 context
->owner
[qual
] = NULL
;
467 ds_put_context(context
);
471 spin_unlock(&ds_lock
);
472 ds_put_context(context
);
476 int ds_request_bts(struct task_struct
*task
, void *base
, size_t size
,
477 ds_ovfl_callback_t ovfl
)
479 return ds_request(task
, base
, size
, ovfl
, ds_bts
);
482 int ds_request_pebs(struct task_struct
*task
, void *base
, size_t size
,
483 ds_ovfl_callback_t ovfl
)
485 return ds_request(task
, base
, size
, ovfl
, ds_pebs
);
488 static int ds_release(struct task_struct
*task
, enum ds_qualifier qual
)
490 struct ds_context
*context
;
493 context
= ds_get_context(task
);
494 error
= ds_validate_access(context
, qual
);
498 kfree(context
->buffer
[qual
]);
499 context
->buffer
[qual
] = NULL
;
501 current
->mm
->total_vm
-= context
->pages
[qual
];
502 current
->mm
->locked_vm
-= context
->pages
[qual
];
503 context
->pages
[qual
] = 0;
504 context
->owner
[qual
] = NULL
;
507 * we put the context twice:
508 * once for the ds_get_context
509 * once for the corresponding ds_request
511 ds_put_context(context
);
513 ds_put_context(context
);
517 int ds_release_bts(struct task_struct
*task
)
519 return ds_release(task
, ds_bts
);
522 int ds_release_pebs(struct task_struct
*task
)
524 return ds_release(task
, ds_pebs
);
527 static int ds_get_index(struct task_struct
*task
, size_t *pos
,
528 enum ds_qualifier qual
)
530 struct ds_context
*context
;
531 unsigned long base
, index
;
534 context
= ds_get_context(task
);
535 error
= ds_validate_access(context
, qual
);
539 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
540 index
= ds_get(context
->ds
, qual
, ds_index
);
542 error
= ((index
- base
) / ds_cfg
.sizeof_rec
[qual
]);
546 ds_put_context(context
);
550 int ds_get_bts_index(struct task_struct
*task
, size_t *pos
)
552 return ds_get_index(task
, pos
, ds_bts
);
555 int ds_get_pebs_index(struct task_struct
*task
, size_t *pos
)
557 return ds_get_index(task
, pos
, ds_pebs
);
560 static int ds_get_end(struct task_struct
*task
, size_t *pos
,
561 enum ds_qualifier qual
)
563 struct ds_context
*context
;
564 unsigned long base
, end
;
567 context
= ds_get_context(task
);
568 error
= ds_validate_access(context
, qual
);
572 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
573 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
575 error
= ((end
- base
) / ds_cfg
.sizeof_rec
[qual
]);
579 ds_put_context(context
);
583 int ds_get_bts_end(struct task_struct
*task
, size_t *pos
)
585 return ds_get_end(task
, pos
, ds_bts
);
588 int ds_get_pebs_end(struct task_struct
*task
, size_t *pos
)
590 return ds_get_end(task
, pos
, ds_pebs
);
593 static int ds_access(struct task_struct
*task
, size_t index
,
594 const void **record
, enum ds_qualifier qual
)
596 struct ds_context
*context
;
597 unsigned long base
, idx
;
603 context
= ds_get_context(task
);
604 error
= ds_validate_access(context
, qual
);
608 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
609 idx
= base
+ (index
* ds_cfg
.sizeof_rec
[qual
]);
612 if (idx
> ds_get(context
->ds
, qual
, ds_absolute_maximum
))
615 *record
= (const void *)idx
;
616 error
= ds_cfg
.sizeof_rec
[qual
];
618 ds_put_context(context
);
622 int ds_access_bts(struct task_struct
*task
, size_t index
, const void **record
)
624 return ds_access(task
, index
, record
, ds_bts
);
627 int ds_access_pebs(struct task_struct
*task
, size_t index
, const void **record
)
629 return ds_access(task
, index
, record
, ds_pebs
);
632 static int ds_write(struct task_struct
*task
, const void *record
, size_t size
,
633 enum ds_qualifier qual
, int force
)
635 struct ds_context
*context
;
642 context
= ds_get_context(task
);
647 error
= ds_validate_access(context
, qual
);
654 unsigned long base
, index
, end
, write_end
, int_th
;
655 unsigned long write_size
, adj_write_size
;
658 * write as much as possible without producing an
659 * overflow interrupt.
661 * interrupt_threshold must either be
662 * - bigger than absolute_maximum or
663 * - point to a record between buffer_base and absolute_maximum
665 * index points to a valid record.
667 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
668 index
= ds_get(context
->ds
, qual
, ds_index
);
669 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
670 int_th
= ds_get(context
->ds
, qual
, ds_interrupt_threshold
);
672 write_end
= min(end
, int_th
);
674 /* if we are already beyond the interrupt threshold,
675 * we fill the entire buffer */
676 if (write_end
<= index
)
679 if (write_end
<= index
)
682 write_size
= min((unsigned long) size
, write_end
- index
);
683 memcpy((void *)index
, record
, write_size
);
685 record
= (const char *)record
+ write_size
;
689 adj_write_size
= write_size
/ ds_cfg
.sizeof_rec
[qual
];
690 adj_write_size
*= ds_cfg
.sizeof_rec
[qual
];
692 /* zero out trailing bytes */
693 memset((char *)index
+ write_size
, 0,
694 adj_write_size
- write_size
);
695 index
+= adj_write_size
;
699 ds_set(context
->ds
, qual
, ds_index
, index
);
702 ds_overflow(task
, context
, qual
);
706 ds_put_context(context
);
710 int ds_write_bts(struct task_struct
*task
, const void *record
, size_t size
)
712 return ds_write(task
, record
, size
, ds_bts
, /* force = */ 0);
715 int ds_write_pebs(struct task_struct
*task
, const void *record
, size_t size
)
717 return ds_write(task
, record
, size
, ds_pebs
, /* force = */ 0);
720 int ds_unchecked_write_bts(struct task_struct
*task
,
721 const void *record
, size_t size
)
723 return ds_write(task
, record
, size
, ds_bts
, /* force = */ 1);
726 int ds_unchecked_write_pebs(struct task_struct
*task
,
727 const void *record
, size_t size
)
729 return ds_write(task
, record
, size
, ds_pebs
, /* force = */ 1);
732 static int ds_reset_or_clear(struct task_struct
*task
,
733 enum ds_qualifier qual
, int clear
)
735 struct ds_context
*context
;
736 unsigned long base
, end
;
739 context
= ds_get_context(task
);
740 error
= ds_validate_access(context
, qual
);
744 base
= ds_get(context
->ds
, qual
, ds_buffer_base
);
745 end
= ds_get(context
->ds
, qual
, ds_absolute_maximum
);
748 memset((void *)base
, 0, end
- base
);
750 ds_set(context
->ds
, qual
, ds_index
, base
);
754 ds_put_context(context
);
758 int ds_reset_bts(struct task_struct
*task
)
760 return ds_reset_or_clear(task
, ds_bts
, /* clear = */ 0);
763 int ds_reset_pebs(struct task_struct
*task
)
765 return ds_reset_or_clear(task
, ds_pebs
, /* clear = */ 0);
768 int ds_clear_bts(struct task_struct
*task
)
770 return ds_reset_or_clear(task
, ds_bts
, /* clear = */ 1);
773 int ds_clear_pebs(struct task_struct
*task
)
775 return ds_reset_or_clear(task
, ds_pebs
, /* clear = */ 1);
778 int ds_get_pebs_reset(struct task_struct
*task
, u64
*value
)
780 struct ds_context
*context
;
786 context
= ds_get_context(task
);
787 error
= ds_validate_access(context
, ds_pebs
);
791 *value
= *(u64
*)(context
->ds
+ (ds_cfg
.sizeof_field
* 8));
795 ds_put_context(context
);
799 int ds_set_pebs_reset(struct task_struct
*task
, u64 value
)
801 struct ds_context
*context
;
804 context
= ds_get_context(task
);
805 error
= ds_validate_access(context
, ds_pebs
);
809 *(u64
*)(context
->ds
+ (ds_cfg
.sizeof_field
* 8)) = value
;
813 ds_put_context(context
);
817 static const struct ds_configuration ds_cfg_var
= {
818 .sizeof_ds
= sizeof(long) * 12,
819 .sizeof_field
= sizeof(long),
820 .sizeof_rec
[ds_bts
] = sizeof(long) * 3,
821 .sizeof_rec
[ds_pebs
] = sizeof(long) * 10
823 static const struct ds_configuration ds_cfg_64
= {
826 .sizeof_rec
[ds_bts
] = 8 * 3,
827 .sizeof_rec
[ds_pebs
] = 8 * 10
831 ds_configure(const struct ds_configuration
*cfg
)
836 void __cpuinit
ds_init_intel(struct cpuinfo_x86
*c
)
840 switch (c
->x86_model
) {
842 case 0xE: /* Pentium M */
843 ds_configure(&ds_cfg_var
);
845 case 0xF: /* Core2 */
846 case 0x1C: /* Atom */
847 ds_configure(&ds_cfg_64
);
850 /* sorry, don't know about them */
855 switch (c
->x86_model
) {
858 case 0x2: /* Netburst */
859 ds_configure(&ds_cfg_var
);
862 /* sorry, don't know about them */
867 /* sorry, don't know about them */
872 void ds_free(struct ds_context
*context
)
874 /* This is called when the task owning the parameter context
875 * is dying. There should not be any user of that context left
876 * to disturb us, anymore. */
877 unsigned long leftovers
= context
->count
;
879 ds_put_context(context
);
881 #endif /* CONFIG_X86_DS */