2 * Copyright (C) 2011-2017, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
4 * Parts came from evlist.c builtin-{top,stat,record}.c, see those files for further
7 * Released under the GPL v2. (and only v2, not any later version)
13 #ifdef HAVE_LIBNUMA_SUPPORT
19 #include "util.h" /* page_size */
21 size_t perf_mmap__mmap_len(struct perf_mmap
*map
)
23 return map
->mask
+ 1 + page_size
;
26 /* When check_messup is true, 'end' must points to a good entry */
27 static union perf_event
*perf_mmap__read(struct perf_mmap
*map
,
30 unsigned char *data
= map
->base
+ page_size
;
31 union perf_event
*event
= NULL
;
32 int diff
= end
- *startp
;
34 if (diff
>= (int)sizeof(event
->header
)) {
37 event
= (union perf_event
*)&data
[*startp
& map
->mask
];
38 size
= event
->header
.size
;
40 if (size
< sizeof(event
->header
) || diff
< (int)size
)
44 * Event straddles the mmap boundary -- header should always
45 * be inside due to u64 alignment of output.
47 if ((*startp
& map
->mask
) + size
!= ((*startp
+ size
) & map
->mask
)) {
48 unsigned int offset
= *startp
;
49 unsigned int len
= min(sizeof(*event
), size
), cpy
;
50 void *dst
= map
->event_copy
;
53 cpy
= min(map
->mask
+ 1 - (offset
& map
->mask
), len
);
54 memcpy(dst
, &data
[offset
& map
->mask
], cpy
);
60 event
= (union perf_event
*)map
->event_copy
;
70 * Read event from ring buffer one by one.
71 * Return one event for each call.
74 * perf_mmap__read_init()
75 * while(event = perf_mmap__read_event()) {
77 * perf_mmap__consume()
79 * perf_mmap__read_done()
81 union perf_event
*perf_mmap__read_event(struct perf_mmap
*map
)
83 union perf_event
*event
;
86 * Check if event was unmapped due to a POLLHUP/POLLERR.
88 if (!refcount_read(&map
->refcnt
))
91 /* non-overwirte doesn't pause the ringbuffer */
93 map
->end
= perf_mmap__read_head(map
);
95 event
= perf_mmap__read(map
, &map
->start
, map
->end
);
98 map
->prev
= map
->start
;
103 static bool perf_mmap__empty(struct perf_mmap
*map
)
105 return perf_mmap__read_head(map
) == map
->prev
&& !map
->auxtrace_mmap
.base
;
108 void perf_mmap__get(struct perf_mmap
*map
)
110 refcount_inc(&map
->refcnt
);
113 void perf_mmap__put(struct perf_mmap
*map
)
115 BUG_ON(map
->base
&& refcount_read(&map
->refcnt
) == 0);
117 if (refcount_dec_and_test(&map
->refcnt
))
118 perf_mmap__munmap(map
);
121 void perf_mmap__consume(struct perf_mmap
*map
)
123 if (!map
->overwrite
) {
126 perf_mmap__write_tail(map
, old
);
129 if (refcount_read(&map
->refcnt
) == 1 && perf_mmap__empty(map
))
133 int __weak
auxtrace_mmap__mmap(struct auxtrace_mmap
*mm __maybe_unused
,
134 struct auxtrace_mmap_params
*mp __maybe_unused
,
135 void *userpg __maybe_unused
,
136 int fd __maybe_unused
)
141 void __weak
auxtrace_mmap__munmap(struct auxtrace_mmap
*mm __maybe_unused
)
145 void __weak
auxtrace_mmap_params__init(struct auxtrace_mmap_params
*mp __maybe_unused
,
146 off_t auxtrace_offset __maybe_unused
,
147 unsigned int auxtrace_pages __maybe_unused
,
148 bool auxtrace_overwrite __maybe_unused
)
152 void __weak
auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params
*mp __maybe_unused
,
153 struct perf_evlist
*evlist __maybe_unused
,
154 int idx __maybe_unused
,
155 bool per_cpu __maybe_unused
)
159 #ifdef HAVE_AIO_SUPPORT
161 #ifdef HAVE_LIBNUMA_SUPPORT
162 static int perf_mmap__aio_alloc(struct perf_mmap
*map
, int idx
)
164 map
->aio
.data
[idx
] = mmap(NULL
, perf_mmap__mmap_len(map
), PROT_READ
|PROT_WRITE
,
165 MAP_PRIVATE
|MAP_ANONYMOUS
, 0, 0);
166 if (map
->aio
.data
[idx
] == MAP_FAILED
) {
167 map
->aio
.data
[idx
] = NULL
;
174 static void perf_mmap__aio_free(struct perf_mmap
*map
, int idx
)
176 if (map
->aio
.data
[idx
]) {
177 munmap(map
->aio
.data
[idx
], perf_mmap__mmap_len(map
));
178 map
->aio
.data
[idx
] = NULL
;
182 static int perf_mmap__aio_bind(struct perf_mmap
*map
, int idx
, int cpu
, int affinity
)
186 unsigned long node_mask
;
188 if (affinity
!= PERF_AFFINITY_SYS
&& cpu__max_node() > 1) {
189 data
= map
->aio
.data
[idx
];
190 mmap_len
= perf_mmap__mmap_len(map
);
191 node_mask
= 1UL << cpu__get_node(cpu
);
192 if (mbind(data
, mmap_len
, MPOL_BIND
, &node_mask
, 1, 0)) {
193 pr_err("Failed to bind [%p-%p] AIO buffer to node %d: error %m\n",
194 data
, data
+ mmap_len
, cpu__get_node(cpu
));
202 static int perf_mmap__aio_alloc(struct perf_mmap
*map
, int idx
)
204 map
->aio
.data
[idx
] = malloc(perf_mmap__mmap_len(map
));
205 if (map
->aio
.data
[idx
] == NULL
)
211 static void perf_mmap__aio_free(struct perf_mmap
*map
, int idx
)
213 zfree(&(map
->aio
.data
[idx
]));
216 static int perf_mmap__aio_bind(struct perf_mmap
*map __maybe_unused
, int idx __maybe_unused
,
217 int cpu __maybe_unused
, int affinity __maybe_unused
)
223 static int perf_mmap__aio_mmap(struct perf_mmap
*map
, struct mmap_params
*mp
)
225 int delta_max
, i
, prio
, ret
;
227 map
->aio
.nr_cblocks
= mp
->nr_cblocks
;
228 if (map
->aio
.nr_cblocks
) {
229 map
->aio
.aiocb
= calloc(map
->aio
.nr_cblocks
, sizeof(struct aiocb
*));
230 if (!map
->aio
.aiocb
) {
231 pr_debug2("failed to allocate aiocb for data buffer, error %m\n");
234 map
->aio
.cblocks
= calloc(map
->aio
.nr_cblocks
, sizeof(struct aiocb
));
235 if (!map
->aio
.cblocks
) {
236 pr_debug2("failed to allocate cblocks for data buffer, error %m\n");
239 map
->aio
.data
= calloc(map
->aio
.nr_cblocks
, sizeof(void *));
240 if (!map
->aio
.data
) {
241 pr_debug2("failed to allocate data buffer, error %m\n");
244 delta_max
= sysconf(_SC_AIO_PRIO_DELTA_MAX
);
245 for (i
= 0; i
< map
->aio
.nr_cblocks
; ++i
) {
246 ret
= perf_mmap__aio_alloc(map
, i
);
248 pr_debug2("failed to allocate data buffer area, error %m");
251 ret
= perf_mmap__aio_bind(map
, i
, map
->cpu
, mp
->affinity
);
255 * Use cblock.aio_fildes value different from -1
256 * to denote started aio write operation on the
257 * cblock so it requires explicit record__aio_sync()
258 * call prior the cblock may be reused again.
260 map
->aio
.cblocks
[i
].aio_fildes
= -1;
262 * Allocate cblocks with priority delta to have
263 * faster aio write system calls because queued requests
264 * are kept in separate per-prio queues and adding
265 * a new request will iterate thru shorter per-prio
266 * list. Blocks with numbers higher than
267 * _SC_AIO_PRIO_DELTA_MAX go with priority 0.
269 prio
= delta_max
- i
;
270 map
->aio
.cblocks
[i
].aio_reqprio
= prio
>= 0 ? prio
: 0;
277 static void perf_mmap__aio_munmap(struct perf_mmap
*map
)
281 for (i
= 0; i
< map
->aio
.nr_cblocks
; ++i
)
282 perf_mmap__aio_free(map
, i
);
284 zfree(&map
->aio
.data
);
285 zfree(&map
->aio
.cblocks
);
286 zfree(&map
->aio
.aiocb
);
289 int perf_mmap__aio_push(struct perf_mmap
*md
, void *to
, int idx
,
290 int push(void *to
, struct aiocb
*cblock
, void *buf
, size_t size
, off_t off
),
293 u64 head
= perf_mmap__read_head(md
);
294 unsigned char *data
= md
->base
+ page_size
;
295 unsigned long size
, size0
= 0;
299 rc
= perf_mmap__read_init(md
);
301 return (rc
== -EAGAIN
) ? 0 : -1;
304 * md->base data is copied into md->data[idx] buffer to
305 * release space in the kernel buffer as fast as possible,
306 * thru perf_mmap__consume() below.
308 * That lets the kernel to proceed with storing more
309 * profiling data into the kernel buffer earlier than other
310 * per-cpu kernel buffers are handled.
312 * Coping can be done in two steps in case the chunk of
313 * profiling data crosses the upper bound of the kernel buffer.
314 * In this case we first move part of data from md->start
315 * till the upper bound and then the reminder from the
316 * beginning of the kernel buffer till the end of
320 size
= md
->end
- md
->start
;
322 if ((md
->start
& md
->mask
) + size
!= (md
->end
& md
->mask
)) {
323 buf
= &data
[md
->start
& md
->mask
];
324 size
= md
->mask
+ 1 - (md
->start
& md
->mask
);
326 memcpy(md
->aio
.data
[idx
], buf
, size
);
330 buf
= &data
[md
->start
& md
->mask
];
331 size
= md
->end
- md
->start
;
333 memcpy(md
->aio
.data
[idx
] + size0
, buf
, size
);
336 * Increment md->refcount to guard md->data[idx] buffer
337 * from premature deallocation because md object can be
338 * released earlier than aio write request started
339 * on mmap->data[idx] is complete.
341 * perf_mmap__put() is done at record__aio_complete()
342 * after started request completion.
347 perf_mmap__consume(md
);
349 rc
= push(to
, &md
->aio
.cblocks
[idx
], md
->aio
.data
[idx
], size0
+ size
, *off
);
351 *off
+= size0
+ size
;
354 * Decrement md->refcount back if aio write
355 * operation failed to start.
363 static int perf_mmap__aio_mmap(struct perf_mmap
*map __maybe_unused
,
364 struct mmap_params
*mp __maybe_unused
)
369 static void perf_mmap__aio_munmap(struct perf_mmap
*map __maybe_unused
)
374 void perf_mmap__munmap(struct perf_mmap
*map
)
376 perf_mmap__aio_munmap(map
);
377 if (map
->base
!= NULL
) {
378 munmap(map
->base
, perf_mmap__mmap_len(map
));
381 refcount_set(&map
->refcnt
, 0);
383 auxtrace_mmap__munmap(&map
->auxtrace_mmap
);
386 static void build_node_mask(int node
, cpu_set_t
*mask
)
389 const struct cpu_map
*cpu_map
= NULL
;
391 cpu_map
= cpu_map__online();
395 nr_cpus
= cpu_map__nr(cpu_map
);
396 for (c
= 0; c
< nr_cpus
; c
++) {
397 cpu
= cpu_map
->map
[c
]; /* map c index to online cpu index */
398 if (cpu__get_node(cpu
) == node
)
403 static void perf_mmap__setup_affinity_mask(struct perf_mmap
*map
, struct mmap_params
*mp
)
405 CPU_ZERO(&map
->affinity_mask
);
406 if (mp
->affinity
== PERF_AFFINITY_NODE
&& cpu__max_node() > 1)
407 build_node_mask(cpu__get_node(map
->cpu
), &map
->affinity_mask
);
408 else if (mp
->affinity
== PERF_AFFINITY_CPU
)
409 CPU_SET(map
->cpu
, &map
->affinity_mask
);
412 int perf_mmap__mmap(struct perf_mmap
*map
, struct mmap_params
*mp
, int fd
, int cpu
)
415 * The last one will be done at perf_mmap__consume(), so that we
416 * make sure we don't prevent tools from consuming every last event in
419 * I.e. we can get the POLLHUP meaning that the fd doesn't exist
420 * anymore, but the last events for it are still in the ring buffer,
421 * waiting to be consumed.
423 * Tools can chose to ignore this at their own discretion, but the
424 * evlist layer can't just drop it when filtering events in
425 * perf_evlist__filter_pollfd().
427 refcount_set(&map
->refcnt
, 2);
429 map
->mask
= mp
->mask
;
430 map
->base
= mmap(NULL
, perf_mmap__mmap_len(map
), mp
->prot
,
432 if (map
->base
== MAP_FAILED
) {
433 pr_debug2("failed to mmap perf event ring buffer, error %d\n",
441 perf_mmap__setup_affinity_mask(map
, mp
);
443 if (auxtrace_mmap__mmap(&map
->auxtrace_mmap
,
444 &mp
->auxtrace_mp
, map
->base
, fd
))
447 return perf_mmap__aio_mmap(map
, mp
);
450 static int overwrite_rb_find_range(void *buf
, int mask
, u64
*start
, u64
*end
)
452 struct perf_event_header
*pheader
;
453 u64 evt_head
= *start
;
456 pr_debug2("%s: buf=%p, start=%"PRIx64
"\n", __func__
, buf
, *start
);
457 pheader
= (struct perf_event_header
*)(buf
+ (*start
& mask
));
459 if (evt_head
- *start
>= (unsigned int)size
) {
460 pr_debug("Finished reading overwrite ring buffer: rewind\n");
461 if (evt_head
- *start
> (unsigned int)size
)
462 evt_head
-= pheader
->size
;
467 pheader
= (struct perf_event_header
*)(buf
+ (evt_head
& mask
));
469 if (pheader
->size
== 0) {
470 pr_debug("Finished reading overwrite ring buffer: get start\n");
475 evt_head
+= pheader
->size
;
476 pr_debug3("move evt_head: %"PRIx64
"\n", evt_head
);
478 WARN_ONCE(1, "Shouldn't get here\n");
483 * Report the start and end of the available data in ringbuffer
485 static int __perf_mmap__read_init(struct perf_mmap
*md
)
487 u64 head
= perf_mmap__read_head(md
);
489 unsigned char *data
= md
->base
+ page_size
;
492 md
->start
= md
->overwrite
? head
: old
;
493 md
->end
= md
->overwrite
? old
: head
;
495 if (md
->start
== md
->end
)
498 size
= md
->end
- md
->start
;
499 if (size
> (unsigned long)(md
->mask
) + 1) {
500 if (!md
->overwrite
) {
501 WARN_ONCE(1, "failed to keep up with mmap data. (warn only once)\n");
504 perf_mmap__consume(md
);
509 * Backward ring buffer is full. We still have a chance to read
510 * most of data from it.
512 if (overwrite_rb_find_range(data
, md
->mask
, &md
->start
, &md
->end
))
519 int perf_mmap__read_init(struct perf_mmap
*map
)
522 * Check if event was unmapped due to a POLLHUP/POLLERR.
524 if (!refcount_read(&map
->refcnt
))
527 return __perf_mmap__read_init(map
);
530 int perf_mmap__push(struct perf_mmap
*md
, void *to
,
531 int push(struct perf_mmap
*map
, void *to
, void *buf
, size_t size
))
533 u64 head
= perf_mmap__read_head(md
);
534 unsigned char *data
= md
->base
+ page_size
;
539 rc
= perf_mmap__read_init(md
);
541 return (rc
== -EAGAIN
) ? 0 : -1;
543 size
= md
->end
- md
->start
;
545 if ((md
->start
& md
->mask
) + size
!= (md
->end
& md
->mask
)) {
546 buf
= &data
[md
->start
& md
->mask
];
547 size
= md
->mask
+ 1 - (md
->start
& md
->mask
);
550 if (push(md
, to
, buf
, size
) < 0) {
556 buf
= &data
[md
->start
& md
->mask
];
557 size
= md
->end
- md
->start
;
560 if (push(md
, to
, buf
, size
) < 0) {
566 perf_mmap__consume(md
);
572 * Mandatory for overwrite mode
573 * The direction of overwrite mode is backward.
574 * The last perf_mmap__read() will set tail to map->prev.
575 * Need to correct the map->prev to head which is the end of next read.
577 void perf_mmap__read_done(struct perf_mmap
*map
)
580 * Check if event was unmapped due to a POLLHUP/POLLERR.
582 if (!refcount_read(&map
->refcnt
))
585 map
->prev
= perf_mmap__read_head(map
);