1 // SPDX-License-Identifier: GPL-2.0
3 * ring buffer tester and benchmark
5 * Copyright (C) 2009 Steven Rostedt <srostedt@redhat.com>
7 #include <linux/ring_buffer.h>
8 #include <linux/completion.h>
9 #include <linux/kthread.h>
10 #include <uapi/linux/sched/types.h>
11 #include <linux/module.h>
12 #include <linux/ktime.h>
13 #include <asm/local.h>
21 /* run time and sleep time in seconds */
22 #define RUN_TIME 10ULL
25 /* number of events for writer to wake up the reader */
26 static int wakeup_interval
= 100;
28 static int reader_finish
;
29 static DECLARE_COMPLETION(read_start
);
30 static DECLARE_COMPLETION(read_done
);
32 static struct trace_buffer
*buffer
;
33 static struct task_struct
*producer
;
34 static struct task_struct
*consumer
;
35 static unsigned long read
;
37 static unsigned int disable_reader
;
38 module_param(disable_reader
, uint
, 0644);
39 MODULE_PARM_DESC(disable_reader
, "only run producer");
41 static unsigned int write_iteration
= 50;
42 module_param(write_iteration
, uint
, 0644);
43 MODULE_PARM_DESC(write_iteration
, "# of writes between timestamp readings");
45 static int producer_nice
= MAX_NICE
;
46 static int consumer_nice
= MAX_NICE
;
48 static int producer_fifo
= -1;
49 static int consumer_fifo
= -1;
51 module_param(producer_nice
, int, 0644);
52 MODULE_PARM_DESC(producer_nice
, "nice prio for producer");
54 module_param(consumer_nice
, int, 0644);
55 MODULE_PARM_DESC(consumer_nice
, "nice prio for consumer");
57 module_param(producer_fifo
, int, 0644);
58 MODULE_PARM_DESC(producer_fifo
, "fifo prio for producer");
60 module_param(consumer_fifo
, int, 0644);
61 MODULE_PARM_DESC(consumer_fifo
, "fifo prio for consumer");
63 static int read_events
;
65 static int test_error
;
67 #define TEST_ERROR() \
80 static bool break_test(void)
82 return test_error
|| kthread_should_stop();
85 static enum event_status
read_event(int cpu
)
87 struct ring_buffer_event
*event
;
91 event
= ring_buffer_consume(buffer
, cpu
, &ts
, NULL
);
95 entry
= ring_buffer_event_data(event
);
105 static enum event_status
read_page(int cpu
)
107 struct ring_buffer_event
*event
;
108 struct rb_page
*rpage
;
109 unsigned long commit
;
116 bpage
= ring_buffer_alloc_read_page(buffer
, cpu
);
118 return EVENT_DROPPED
;
120 ret
= ring_buffer_read_page(buffer
, &bpage
, PAGE_SIZE
, cpu
, 1);
123 /* The commit may have missed event flags set, clear them */
124 commit
= local_read(&rpage
->commit
) & 0xfffff;
125 for (i
= 0; i
< commit
&& !test_error
; i
+= inc
) {
127 if (i
>= (PAGE_SIZE
- offsetof(struct rb_page
, data
))) {
133 event
= (void *)&rpage
->data
[i
];
134 switch (event
->type_len
) {
135 case RINGBUF_TYPE_PADDING
:
136 /* failed writes may be discarded events */
137 if (!event
->time_delta
)
139 inc
= event
->array
[0] + 4;
141 case RINGBUF_TYPE_TIME_EXTEND
:
145 entry
= ring_buffer_event_data(event
);
151 if (!event
->array
[0]) {
155 inc
= event
->array
[0] + 4;
158 entry
= ring_buffer_event_data(event
);
164 inc
= ((event
->type_len
+ 1) * 4);
175 ring_buffer_free_read_page(buffer
, cpu
, bpage
);
178 return EVENT_DROPPED
;
182 static void ring_buffer_consumer(void)
184 /* toggle between reading pages and events */
189 * Continue running until the producer specifically asks to stop
190 * and is ready for the completion.
192 while (!READ_ONCE(reader_finish
)) {
195 while (found
&& !test_error
) {
199 for_each_online_cpu(cpu
) {
200 enum event_status stat
;
203 stat
= read_event(cpu
);
205 stat
= read_page(cpu
);
210 if (stat
== EVENT_FOUND
)
216 /* Wait till the producer wakes us up when there is more data
217 * available or when the producer wants us to finish reading.
219 set_current_state(TASK_INTERRUPTIBLE
);
225 __set_current_state(TASK_RUNNING
);
227 complete(&read_done
);
230 static void ring_buffer_producer(void)
232 ktime_t start_time
, end_time
, timeout
;
233 unsigned long long time
;
234 unsigned long long entries
;
235 unsigned long long overruns
;
236 unsigned long missed
= 0;
237 unsigned long hit
= 0;
242 * Hammer the buffer for 10 secs (this may
243 * make the system stall)
245 trace_printk("Starting ring buffer hammer\n");
246 start_time
= ktime_get();
247 timeout
= ktime_add_ns(start_time
, RUN_TIME
* NSEC_PER_SEC
);
249 struct ring_buffer_event
*event
;
253 for (i
= 0; i
< write_iteration
; i
++) {
254 event
= ring_buffer_lock_reserve(buffer
, 10);
259 entry
= ring_buffer_event_data(event
);
260 *entry
= smp_processor_id();
261 ring_buffer_unlock_commit(buffer
, event
);
264 end_time
= ktime_get();
267 if (consumer
&& !(cnt
% wakeup_interval
))
268 wake_up_process(consumer
);
270 #ifndef CONFIG_PREEMPTION
272 * If we are a non preempt kernel, the 10 seconds run will
273 * stop everything while it runs. Instead, we will call
274 * cond_resched and also add any time that was lost by a
277 * Do a cond resched at the same frequency we would wake up
280 if (cnt
% wakeup_interval
)
283 } while (ktime_before(end_time
, timeout
) && !break_test());
284 trace_printk("End ring buffer hammer\n");
287 /* Init both completions here to avoid races */
288 init_completion(&read_start
);
289 init_completion(&read_done
);
290 /* the completions must be visible before the finish var */
293 wake_up_process(consumer
);
294 wait_for_completion(&read_done
);
297 time
= ktime_us_delta(end_time
, start_time
);
299 entries
= ring_buffer_entries(buffer
);
300 overruns
= ring_buffer_overruns(buffer
);
303 trace_printk("ERROR!\n");
305 if (!disable_reader
) {
306 if (consumer_fifo
< 0)
307 trace_printk("Running Consumer at nice: %d\n",
310 trace_printk("Running Consumer at SCHED_FIFO %d\n",
313 if (producer_fifo
< 0)
314 trace_printk("Running Producer at nice: %d\n",
317 trace_printk("Running Producer at SCHED_FIFO %d\n",
320 /* Let the user know that the test is running at low priority */
321 if (producer_fifo
< 0 && consumer_fifo
< 0 &&
322 producer_nice
== MAX_NICE
&& consumer_nice
== MAX_NICE
)
323 trace_printk("WARNING!!! This test is running at lowest priority.\n");
325 trace_printk("Time: %lld (usecs)\n", time
);
326 trace_printk("Overruns: %lld\n", overruns
);
328 trace_printk("Read: (reader disabled)\n");
330 trace_printk("Read: %ld (by %s)\n", read
,
331 read_events
? "events" : "pages");
332 trace_printk("Entries: %lld\n", entries
);
333 trace_printk("Total: %lld\n", entries
+ overruns
+ read
);
334 trace_printk("Missed: %ld\n", missed
);
335 trace_printk("Hit: %ld\n", hit
);
337 /* Convert time from usecs to millisecs */
338 do_div(time
, USEC_PER_MSEC
);
342 trace_printk("TIME IS ZERO??\n");
344 trace_printk("Entries per millisec: %ld\n", hit
);
347 /* Calculate the average time in nanosecs */
348 avg
= NSEC_PER_MSEC
/ hit
;
349 trace_printk("%ld ns per entry\n", avg
);
354 missed
/= (long)time
;
356 trace_printk("Total iterations per millisec: %ld\n",
359 /* it is possible that hit + missed will overflow and be zero */
360 if (!(hit
+ missed
)) {
361 trace_printk("hit + missed overflowed and totalled zero!\n");
362 hit
--; /* make it non zero */
365 /* Calculate the average time in nanosecs */
366 avg
= NSEC_PER_MSEC
/ (hit
+ missed
);
367 trace_printk("%ld ns per entry\n", avg
);
371 static void wait_to_die(void)
373 set_current_state(TASK_INTERRUPTIBLE
);
374 while (!kthread_should_stop()) {
376 set_current_state(TASK_INTERRUPTIBLE
);
378 __set_current_state(TASK_RUNNING
);
381 static int ring_buffer_consumer_thread(void *arg
)
383 while (!break_test()) {
384 complete(&read_start
);
386 ring_buffer_consumer();
388 set_current_state(TASK_INTERRUPTIBLE
);
393 __set_current_state(TASK_RUNNING
);
395 if (!kthread_should_stop())
401 static int ring_buffer_producer_thread(void *arg
)
403 while (!break_test()) {
404 ring_buffer_reset(buffer
);
407 wake_up_process(consumer
);
408 wait_for_completion(&read_start
);
411 ring_buffer_producer();
415 trace_printk("Sleeping for 10 secs\n");
416 set_current_state(TASK_INTERRUPTIBLE
);
419 schedule_timeout(HZ
* SLEEP_TIME
);
423 __set_current_state(TASK_RUNNING
);
424 if (!kthread_should_stop())
430 static int __init
ring_buffer_benchmark_init(void)
434 /* make a one meg buffer in overwite mode */
435 buffer
= ring_buffer_alloc(1000000, RB_FL_OVERWRITE
);
439 if (!disable_reader
) {
440 consumer
= kthread_create(ring_buffer_consumer_thread
,
441 NULL
, "rb_consumer");
442 ret
= PTR_ERR(consumer
);
443 if (IS_ERR(consumer
))
447 producer
= kthread_run(ring_buffer_producer_thread
,
448 NULL
, "rb_producer");
449 ret
= PTR_ERR(producer
);
451 if (IS_ERR(producer
))
455 * Run them as low-prio background tasks by default:
457 if (!disable_reader
) {
458 if (consumer_fifo
>= 0) {
459 struct sched_param param
= {
460 .sched_priority
= consumer_fifo
462 sched_setscheduler(consumer
, SCHED_FIFO
, ¶m
);
464 set_user_nice(consumer
, consumer_nice
);
467 if (producer_fifo
>= 0) {
468 struct sched_param param
= {
469 .sched_priority
= producer_fifo
471 sched_setscheduler(producer
, SCHED_FIFO
, ¶m
);
473 set_user_nice(producer
, producer_nice
);
479 kthread_stop(consumer
);
482 ring_buffer_free(buffer
);
486 static void __exit
ring_buffer_benchmark_exit(void)
488 kthread_stop(producer
);
490 kthread_stop(consumer
);
491 ring_buffer_free(buffer
);
494 module_init(ring_buffer_benchmark_init
);
495 module_exit(ring_buffer_benchmark_exit
);
497 MODULE_AUTHOR("Steven Rostedt");
498 MODULE_DESCRIPTION("ring_buffer_benchmark");
499 MODULE_LICENSE("GPL");