?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm thin metadata: fix __udivdi3 undefined on 32-bit
[linux/fpc-iii.git] / tools / perf / bench / numa.c
blob73d192f57dc3429be1cb55a51cf59e29c8eebba1
1 /*
2 * numa.c
3 *
4 * numa: Simulate NUMA-sensitive workload and measure their NUMA performance
5 */
6
7 #include "../perf.h"
8 #include "../builtin.h"
9 #include "../util/util.h"
10 #include "../util/parse-options.h"
11 #include "../util/cloexec.h"
12
13 #include "bench.h"
14
15 #include <errno.h>
16 #include <sched.h>
17 #include <stdio.h>
18 #include <assert.h>
19 #include <malloc.h>
20 #include <signal.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <pthread.h>
25 #include <sys/mman.h>
26 #include <sys/time.h>
27 #include <sys/resource.h>
28 #include <sys/wait.h>
29 #include <sys/prctl.h>
30 #include <sys/types.h>
31
32 #include <numa.h>
33 #include <numaif.h>
34
35 /*
36 * Regular printout to the terminal, supressed if -q is specified:
37 */
38 #define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0)
39
40 /*
41 * Debug printf:
42 */
43 #define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0)
44
45 struct thread_data {
46 int curr_cpu;
47 cpu_set_t bind_cpumask;
48 int bind_node;
49 u8 *process_data;
50 int process_nr;
51 int thread_nr;
52 int task_nr;
53 unsigned int loops_done;
54 u64 val;
55 u64 runtime_ns;
56 u64 system_time_ns;
57 u64 user_time_ns;
58 double speed_gbs;
59 pthread_mutex_t *process_lock;
60 };
61
62 /* Parameters set by options: */
63
64 struct params {
65 /* Startup synchronization: */
66 bool serialize_startup;
67
68 /* Task hierarchy: */
69 int nr_proc;
70 int nr_threads;
71
72 /* Working set sizes: */
73 const char *mb_global_str;
74 const char *mb_proc_str;
75 const char *mb_proc_locked_str;
76 const char *mb_thread_str;
77
78 double mb_global;
79 double mb_proc;
80 double mb_proc_locked;
81 double mb_thread;
82
83 /* Access patterns to the working set: */
84 bool data_reads;
85 bool data_writes;
86 bool data_backwards;
87 bool data_zero_memset;
88 bool data_rand_walk;
89 u32 nr_loops;
90 u32 nr_secs;
91 u32 sleep_usecs;
92
93 /* Working set initialization: */
94 bool init_zero;
95 bool init_random;
96 bool init_cpu0;
97
98 /* Misc options: */
99 int show_details;
100 int run_all;
101 int thp;
102
103 long bytes_global;
104 long bytes_process;
105 long bytes_process_locked;
106 long bytes_thread;
107
108 int nr_tasks;
109 bool show_quiet;
110
111 bool show_convergence;
112 bool measure_convergence;
113
114 int perturb_secs;
115 int nr_cpus;
116 int nr_nodes;
117
118 /* Affinity options -C and -N: */
119 char *cpu_list_str;
120 char *node_list_str;
121 };
122
123
124 /* Global, read-writable area, accessible to all processes and threads: */
125
126 struct global_info {
127 u8 *data;
128
129 pthread_mutex_t startup_mutex;
130 int nr_tasks_started;
131
132 pthread_mutex_t startup_done_mutex;
133
134 pthread_mutex_t start_work_mutex;
135 int nr_tasks_working;
136
137 pthread_mutex_t stop_work_mutex;
138 u64 bytes_done;
139
140 struct thread_data *threads;
141
142 /* Convergence latency measurement: */
143 bool all_converged;
144 bool stop_work;
145
146 int print_once;
147
148 struct params p;
149 };
150
151 static struct global_info *g = NULL;
152
153 static int parse_cpus_opt(const struct option *opt, const char *arg, int unset);
154 static int parse_nodes_opt(const struct option *opt, const char *arg, int unset);
155
156 struct params p0;
157
158 static const struct option options[] = {
159 OPT_INTEGER('p', "nr_proc" , &p0.nr_proc, "number of processes"),
160 OPT_INTEGER('t', "nr_threads" , &p0.nr_threads, "number of threads per process"),
161
162 OPT_STRING('G', "mb_global" , &p0.mb_global_str, "MB", "global memory (MBs)"),
163 OPT_STRING('P', "mb_proc" , &p0.mb_proc_str, "MB", "process memory (MBs)"),
164 OPT_STRING('L', "mb_proc_locked", &p0.mb_proc_locked_str,"MB", "process serialized/locked memory access (MBs), <= process_memory"),
165 OPT_STRING('T', "mb_thread" , &p0.mb_thread_str, "MB", "thread memory (MBs)"),
166
167 OPT_UINTEGER('l', "nr_loops" , &p0.nr_loops, "max number of loops to run (default: unlimited)"),
168 OPT_UINTEGER('s', "nr_secs" , &p0.nr_secs, "max number of seconds to run (default: 5 secs)"),
169 OPT_UINTEGER('u', "usleep" , &p0.sleep_usecs, "usecs to sleep per loop iteration"),
170
171 OPT_BOOLEAN('R', "data_reads" , &p0.data_reads, "access the data via writes (can be mixed with -W)"),
172 OPT_BOOLEAN('W', "data_writes" , &p0.data_writes, "access the data via writes (can be mixed with -R)"),
173 OPT_BOOLEAN('B', "data_backwards", &p0.data_backwards, "access the data backwards as well"),
174 OPT_BOOLEAN('Z', "data_zero_memset", &p0.data_zero_memset,"access the data via glibc bzero only"),
175 OPT_BOOLEAN('r', "data_rand_walk", &p0.data_rand_walk, "access the data with random (32bit LFSR) walk"),
176
177
178 OPT_BOOLEAN('z', "init_zero" , &p0.init_zero, "bzero the initial allocations"),
179 OPT_BOOLEAN('I', "init_random" , &p0.init_random, "randomize the contents of the initial allocations"),
180 OPT_BOOLEAN('0', "init_cpu0" , &p0.init_cpu0, "do the initial allocations on CPU#0"),
181 OPT_INTEGER('x', "perturb_secs", &p0.perturb_secs, "perturb thread 0/0 every X secs, to test convergence stability"),
182
183 OPT_INCR ('d', "show_details" , &p0.show_details, "Show details"),
184 OPT_INCR ('a', "all" , &p0.run_all, "Run all tests in the suite"),
185 OPT_INTEGER('H', "thp" , &p0.thp, "MADV_NOHUGEPAGE < 0 < MADV_HUGEPAGE"),
186 OPT_BOOLEAN('c', "show_convergence", &p0.show_convergence, "show convergence details"),
187 OPT_BOOLEAN('m', "measure_convergence", &p0.measure_convergence, "measure convergence latency"),
188 OPT_BOOLEAN('q', "quiet" , &p0.show_quiet, "quiet mode"),
189 OPT_BOOLEAN('S', "serialize-startup", &p0.serialize_startup,"serialize thread startup"),
190
191 /* Special option string parsing callbacks: */
192 OPT_CALLBACK('C', "cpus", NULL, "cpu[,cpu2,...cpuN]",
193 "bind the first N tasks to these specific cpus (the rest is unbound)",
194 parse_cpus_opt),
195 OPT_CALLBACK('M', "memnodes", NULL, "node[,node2,...nodeN]",
196 "bind the first N tasks to these specific memory nodes (the rest is unbound)",
197 parse_nodes_opt),
198 OPT_END()
199 };
200
201 static const char * const bench_numa_usage[] = {
202 "perf bench numa <options>",
203 NULL
204 };
205
206 static const char * const numa_usage[] = {
207 "perf bench numa mem [<options>]",
208 NULL
209 };
210
211 /*
212 * To get number of numa nodes present.
213 */
214 static int nr_numa_nodes(void)
215 {
216 int i, nr_nodes = 0;
217
218 for (i = 0; i < g->p.nr_nodes; i++) {
219 if (numa_bitmask_isbitset(numa_nodes_ptr, i))
220 nr_nodes++;
221 }
222
223 return nr_nodes;
224 }
225
226 /*
227 * To check if given numa node is present.
228 */
229 static int is_node_present(int node)
230 {
231 return numa_bitmask_isbitset(numa_nodes_ptr, node);
232 }
233
234 /*
235 * To check given numa node has cpus.
236 */
237 static bool node_has_cpus(int node)
238 {
239 struct bitmask *cpu = numa_allocate_cpumask();
240 unsigned int i;
241
242 if (cpu && !numa_node_to_cpus(node, cpu)) {
243 for (i = 0; i < cpu->size; i++) {
244 if (numa_bitmask_isbitset(cpu, i))
245 return true;
246 }
247 }
248
249 return false; /* lets fall back to nocpus safely */
250 }
251
252 static cpu_set_t bind_to_cpu(int target_cpu)
253 {
254 cpu_set_t orig_mask, mask;
255 int ret;
256
257 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask);
258 BUG_ON(ret);
259
260 CPU_ZERO(&mask);
261
262 if (target_cpu == -1) {
263 int cpu;
264
265 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
266 CPU_SET(cpu, &mask);
267 } else {
268 BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus);
269 CPU_SET(target_cpu, &mask);
270 }
271
272 ret = sched_setaffinity(0, sizeof(mask), &mask);
273 BUG_ON(ret);
274
275 return orig_mask;
276 }
277
278 static cpu_set_t bind_to_node(int target_node)
279 {
280 int cpus_per_node = g->p.nr_cpus / nr_numa_nodes();
281 cpu_set_t orig_mask, mask;
282 int cpu;
283 int ret;
284
285 BUG_ON(cpus_per_node * nr_numa_nodes() != g->p.nr_cpus);
286 BUG_ON(!cpus_per_node);
287
288 ret = sched_getaffinity(0, sizeof(orig_mask), &orig_mask);
289 BUG_ON(ret);
290
291 CPU_ZERO(&mask);
292
293 if (target_node == -1) {
294 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
295 CPU_SET(cpu, &mask);
296 } else {
297 int cpu_start = (target_node + 0) * cpus_per_node;
298 int cpu_stop = (target_node + 1) * cpus_per_node;
299
300 BUG_ON(cpu_stop > g->p.nr_cpus);
301
302 for (cpu = cpu_start; cpu < cpu_stop; cpu++)
303 CPU_SET(cpu, &mask);
304 }
305
306 ret = sched_setaffinity(0, sizeof(mask), &mask);
307 BUG_ON(ret);
308
309 return orig_mask;
310 }
311
312 static void bind_to_cpumask(cpu_set_t mask)
313 {
314 int ret;
315
316 ret = sched_setaffinity(0, sizeof(mask), &mask);
317 BUG_ON(ret);
318 }
319
320 static void mempol_restore(void)
321 {
322 int ret;
323
324 ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1);
325
326 BUG_ON(ret);
327 }
328
329 static void bind_to_memnode(int node)
330 {
331 unsigned long nodemask;
332 int ret;
333
334 if (node == -1)
335 return;
336
337 BUG_ON(g->p.nr_nodes > (int)sizeof(nodemask));
338 nodemask = 1L << node;
339
340 ret = set_mempolicy(MPOL_BIND, &nodemask, sizeof(nodemask)*8);
341 dprintf("binding to node %d, mask: %016lx => %d\n", node, nodemask, ret);
342
343 BUG_ON(ret);
344 }
345
346 #define HPSIZE (2*1024*1024)
347
348 #define set_taskname(fmt...) \
349 do { \
350 char name[20]; \
351 \
352 snprintf(name, 20, fmt); \
353 prctl(PR_SET_NAME, name); \
354 } while (0)
355
356 static u8 *alloc_data(ssize_t bytes0, int map_flags,
357 int init_zero, int init_cpu0, int thp, int init_random)
358 {
359 cpu_set_t orig_mask;
360 ssize_t bytes;
361 u8 *buf;
362 int ret;
363
364 if (!bytes0)
365 return NULL;
366
367 /* Allocate and initialize all memory on CPU#0: */
368 if (init_cpu0) {
369 orig_mask = bind_to_node(0);
370 bind_to_memnode(0);
371 }
372
373 bytes = bytes0 + HPSIZE;
374
375 buf = (void *)mmap(0, bytes, PROT_READ|PROT_WRITE, MAP_ANON|map_flags, -1, 0);
376 BUG_ON(buf == (void *)-1);
377
378 if (map_flags == MAP_PRIVATE) {
379 if (thp > 0) {
380 ret = madvise(buf, bytes, MADV_HUGEPAGE);
381 if (ret && !g->print_once) {
382 g->print_once = 1;
383 printf("WARNING: Could not enable THP - do: 'echo madvise > /sys/kernel/mm/transparent_hugepage/enabled'\n");
384 }
385 }
386 if (thp < 0) {
387 ret = madvise(buf, bytes, MADV_NOHUGEPAGE);
388 if (ret && !g->print_once) {
389 g->print_once = 1;
390 printf("WARNING: Could not disable THP: run a CONFIG_TRANSPARENT_HUGEPAGE kernel?\n");
391 }
392 }
393 }
394
395 if (init_zero) {
396 bzero(buf, bytes);
397 } else {
398 /* Initialize random contents, different in each word: */
399 if (init_random) {
400 u64 *wbuf = (void *)buf;
401 long off = rand();
402 long i;
403
404 for (i = 0; i < bytes/8; i++)
405 wbuf[i] = i + off;
406 }
407 }
408
409 /* Align to 2MB boundary: */
410 buf = (void *)(((unsigned long)buf + HPSIZE-1) & ~(HPSIZE-1));
411
412 /* Restore affinity: */
413 if (init_cpu0) {
414 bind_to_cpumask(orig_mask);
415 mempol_restore();
416 }
417
418 return buf;
419 }
420
421 static void free_data(void *data, ssize_t bytes)
422 {
423 int ret;
424
425 if (!data)
426 return;
427
428 ret = munmap(data, bytes);
429 BUG_ON(ret);
430 }
431
432 /*
433 * Create a shared memory buffer that can be shared between processes, zeroed:
434 */
435 static void * zalloc_shared_data(ssize_t bytes)
436 {
437 return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0, g->p.thp, g->p.init_random);
438 }
439
440 /*
441 * Create a shared memory buffer that can be shared between processes:
442 */
443 static void * setup_shared_data(ssize_t bytes)
444 {
445 return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0, g->p.thp, g->p.init_random);
446 }
447
448 /*
449 * Allocate process-local memory - this will either be shared between
450 * threads of this process, or only be accessed by this thread:
451 */
452 static void * setup_private_data(ssize_t bytes)
453 {
454 return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random);
455 }
456
457 /*
458 * Return a process-shared (global) mutex:
459 */
460 static void init_global_mutex(pthread_mutex_t *mutex)
461 {
462 pthread_mutexattr_t attr;
463
464 pthread_mutexattr_init(&attr);
465 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
466 pthread_mutex_init(mutex, &attr);
467 }
468
469 static int parse_cpu_list(const char *arg)
470 {
471 p0.cpu_list_str = strdup(arg);
472
473 dprintf("got CPU list: {%s}\n", p0.cpu_list_str);
474
475 return 0;
476 }
477
478 static int parse_setup_cpu_list(void)
479 {
480 struct thread_data *td;
481 char *str0, *str;
482 int t;
483
484 if (!g->p.cpu_list_str)
485 return 0;
486
487 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);
488
489 str0 = str = strdup(g->p.cpu_list_str);
490 t = 0;
491
492 BUG_ON(!str);
493
494 tprintf("# binding tasks to CPUs:\n");
495 tprintf("# ");
496
497 while (true) {
498 int bind_cpu, bind_cpu_0, bind_cpu_1;
499 char *tok, *tok_end, *tok_step, *tok_len, *tok_mul;
500 int bind_len;
501 int step;
502 int mul;
503
504 tok = strsep(&str, ",");
505 if (!tok)
506 break;
507
508 tok_end = strstr(tok, "-");
509
510 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end);
511 if (!tok_end) {
512 /* Single CPU specified: */
513 bind_cpu_0 = bind_cpu_1 = atol(tok);
514 } else {
515 /* CPU range specified (for example: "5-11"): */
516 bind_cpu_0 = atol(tok);
517 bind_cpu_1 = atol(tok_end + 1);
518 }
519
520 step = 1;
521 tok_step = strstr(tok, "#");
522 if (tok_step) {
523 step = atol(tok_step + 1);
524 BUG_ON(step <= 0 || step >= g->p.nr_cpus);
525 }
526
527 /*
528 * Mask length.
529 * Eg: "--cpus 8_4-16#4" means: '--cpus 8_4,12_4,16_4',
530 * where the _4 means the next 4 CPUs are allowed.
531 */
532 bind_len = 1;
533 tok_len = strstr(tok, "_");
534 if (tok_len) {
535 bind_len = atol(tok_len + 1);
536 BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus);
537 }
538
539 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */
540 mul = 1;
541 tok_mul = strstr(tok, "x");
542 if (tok_mul) {
543 mul = atol(tok_mul + 1);
544 BUG_ON(mul <= 0);
545 }
546
547 dprintf("CPUs: %d_%d-%d#%dx%d\n", bind_cpu_0, bind_len, bind_cpu_1, step, mul);
548
549 if (bind_cpu_0 >= g->p.nr_cpus || bind_cpu_1 >= g->p.nr_cpus) {
550 printf("\nTest not applicable, system has only %d CPUs.\n", g->p.nr_cpus);
551 return -1;
552 }
553
554 BUG_ON(bind_cpu_0 < 0 || bind_cpu_1 < 0);
555 BUG_ON(bind_cpu_0 > bind_cpu_1);
556
557 for (bind_cpu = bind_cpu_0; bind_cpu <= bind_cpu_1; bind_cpu += step) {
558 int i;
559
560 for (i = 0; i < mul; i++) {
561 int cpu;
562
563 if (t >= g->p.nr_tasks) {
564 printf("\n# NOTE: ignoring bind CPUs starting at CPU#%d\n #", bind_cpu);
565 goto out;
566 }
567 td = g->threads + t;
568
569 if (t)
570 tprintf(",");
571 if (bind_len > 1) {
572 tprintf("%2d/%d", bind_cpu, bind_len);
573 } else {
574 tprintf("%2d", bind_cpu);
575 }
576
577 CPU_ZERO(&td->bind_cpumask);
578 for (cpu = bind_cpu; cpu < bind_cpu+bind_len; cpu++) {
579 BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus);
580 CPU_SET(cpu, &td->bind_cpumask);
581 }
582 t++;
583 }
584 }
585 }
586 out:
587
588 tprintf("\n");
589
590 if (t < g->p.nr_tasks)
591 printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t);
592
593 free(str0);
594 return 0;
595 }
596
597 static int parse_cpus_opt(const struct option *opt __maybe_unused,
598 const char *arg, int unset __maybe_unused)
599 {
600 if (!arg)
601 return -1;
602
603 return parse_cpu_list(arg);
604 }
605
606 static int parse_node_list(const char *arg)
607 {
608 p0.node_list_str = strdup(arg);
609
610 dprintf("got NODE list: {%s}\n", p0.node_list_str);
611
612 return 0;
613 }
614
615 static int parse_setup_node_list(void)
616 {
617 struct thread_data *td;
618 char *str0, *str;
619 int t;
620
621 if (!g->p.node_list_str)
622 return 0;
623
624 dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);
625
626 str0 = str = strdup(g->p.node_list_str);
627 t = 0;
628
629 BUG_ON(!str);
630
631 tprintf("# binding tasks to NODEs:\n");
632 tprintf("# ");
633
634 while (true) {
635 int bind_node, bind_node_0, bind_node_1;
636 char *tok, *tok_end, *tok_step, *tok_mul;
637 int step;
638 int mul;
639
640 tok = strsep(&str, ",");
641 if (!tok)
642 break;
643
644 tok_end = strstr(tok, "-");
645
646 dprintf("\ntoken: {%s}, end: {%s}\n", tok, tok_end);
647 if (!tok_end) {
648 /* Single NODE specified: */
649 bind_node_0 = bind_node_1 = atol(tok);
650 } else {
651 /* NODE range specified (for example: "5-11"): */
652 bind_node_0 = atol(tok);
653 bind_node_1 = atol(tok_end + 1);
654 }
655
656 step = 1;
657 tok_step = strstr(tok, "#");
658 if (tok_step) {
659 step = atol(tok_step + 1);
660 BUG_ON(step <= 0 || step >= g->p.nr_nodes);
661 }
662
663 /* Multiplicator shortcut, "0x8" is a shortcut for: "0,0,0,0,0,0,0,0" */
664 mul = 1;
665 tok_mul = strstr(tok, "x");
666 if (tok_mul) {
667 mul = atol(tok_mul + 1);
668 BUG_ON(mul <= 0);
669 }
670
671 dprintf("NODEs: %d-%d #%d\n", bind_node_0, bind_node_1, step);
672
673 if (bind_node_0 >= g->p.nr_nodes || bind_node_1 >= g->p.nr_nodes) {
674 printf("\nTest not applicable, system has only %d nodes.\n", g->p.nr_nodes);
675 return -1;
676 }
677
678 BUG_ON(bind_node_0 < 0 || bind_node_1 < 0);
679 BUG_ON(bind_node_0 > bind_node_1);
680
681 for (bind_node = bind_node_0; bind_node <= bind_node_1; bind_node += step) {
682 int i;
683
684 for (i = 0; i < mul; i++) {
685 if (t >= g->p.nr_tasks || !node_has_cpus(bind_node)) {
686 printf("\n# NOTE: ignoring bind NODEs starting at NODE#%d\n", bind_node);
687 goto out;
688 }
689 td = g->threads + t;
690
691 if (!t)
692 tprintf(" %2d", bind_node);
693 else
694 tprintf(",%2d", bind_node);
695
696 td->bind_node = bind_node;
697 t++;
698 }
699 }
700 }
701 out:
702
703 tprintf("\n");
704
705 if (t < g->p.nr_tasks)
706 printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t);
707
708 free(str0);
709 return 0;
710 }
711
712 static int parse_nodes_opt(const struct option *opt __maybe_unused,
713 const char *arg, int unset __maybe_unused)
714 {
715 if (!arg)
716 return -1;
717
718 return parse_node_list(arg);
719
720 return 0;
721 }
722
723 #define BIT(x) (1ul << x)
724
725 static inline uint32_t lfsr_32(uint32_t lfsr)
726 {
727 const uint32_t taps = BIT(1) | BIT(5) | BIT(6) | BIT(31);
728 return (lfsr>>1) ^ ((0x0u - (lfsr & 0x1u)) & taps);
729 }
730
731 /*
732 * Make sure there's real data dependency to RAM (when read
733 * accesses are enabled), so the compiler, the CPU and the
734 * kernel (KSM, zero page, etc.) cannot optimize away RAM
735 * accesses:
736 */
737 static inline u64 access_data(u64 *data __attribute__((unused)), u64 val)
738 {
739 if (g->p.data_reads)
740 val += *data;
741 if (g->p.data_writes)
742 *data = val + 1;
743 return val;
744 }
745
746 /*
747 * The worker process does two types of work, a forwards going
748 * loop and a backwards going loop.
749 *
750 * We do this so that on multiprocessor systems we do not create
751 * a 'train' of processing, with highly synchronized processes,
752 * skewing the whole benchmark.
753 */
754 static u64 do_work(u8 *__data, long bytes, int nr, int nr_max, int loop, u64 val)
755 {
756 long words = bytes/sizeof(u64);
757 u64 *data = (void *)__data;
758 long chunk_0, chunk_1;
759 u64 *d0, *d, *d1;
760 long off;
761 long i;
762
763 BUG_ON(!data && words);
764 BUG_ON(data && !words);
765
766 if (!data)
767 return val;
768
769 /* Very simple memset() work variant: */
770 if (g->p.data_zero_memset && !g->p.data_rand_walk) {
771 bzero(data, bytes);
772 return val;
773 }
774
775 /* Spread out by PID/TID nr and by loop nr: */
776 chunk_0 = words/nr_max;
777 chunk_1 = words/g->p.nr_loops;
778 off = nr*chunk_0 + loop*chunk_1;
779
780 while (off >= words)
781 off -= words;
782
783 if (g->p.data_rand_walk) {
784 u32 lfsr = nr + loop + val;
785 int j;
786
787 for (i = 0; i < words/1024; i++) {
788 long start, end;
789
790 lfsr = lfsr_32(lfsr);
791
792 start = lfsr % words;
793 end = min(start + 1024, words-1);
794
795 if (g->p.data_zero_memset) {
796 bzero(data + start, (end-start) * sizeof(u64));
797 } else {
798 for (j = start; j < end; j++)
799 val = access_data(data + j, val);
800 }
801 }
802 } else if (!g->p.data_backwards || (nr + loop) & 1) {
803
804 d0 = data + off;
805 d = data + off + 1;
806 d1 = data + words;
807
808 /* Process data forwards: */
809 for (;;) {
810 if (unlikely(d >= d1))
811 d = data;
812 if (unlikely(d == d0))
813 break;
814
815 val = access_data(d, val);
816
817 d++;
818 }
819 } else {
820 /* Process data backwards: */
821
822 d0 = data + off;
823 d = data + off - 1;
824 d1 = data + words;
825
826 /* Process data forwards: */
827 for (;;) {
828 if (unlikely(d < data))
829 d = data + words-1;
830 if (unlikely(d == d0))
831 break;
832
833 val = access_data(d, val);
834
835 d--;
836 }
837 }
838
839 return val;
840 }
841
842 static void update_curr_cpu(int task_nr, unsigned long bytes_worked)
843 {
844 unsigned int cpu;
845
846 cpu = sched_getcpu();
847
848 g->threads[task_nr].curr_cpu = cpu;
849 prctl(0, bytes_worked);
850 }
851
852 #define MAX_NR_NODES 64
853
854 /*
855 * Count the number of nodes a process's threads
856 * are spread out on.
857 *
858 * A count of 1 means that the process is compressed
859 * to a single node. A count of g->p.nr_nodes means it's
860 * spread out on the whole system.
861 */
862 static int count_process_nodes(int process_nr)
863 {
864 char node_present[MAX_NR_NODES] = { 0, };
865 int nodes;
866 int n, t;
867
868 for (t = 0; t < g->p.nr_threads; t++) {
869 struct thread_data *td;
870 int task_nr;
871 int node;
872
873 task_nr = process_nr*g->p.nr_threads + t;
874 td = g->threads + task_nr;
875
876 node = numa_node_of_cpu(td->curr_cpu);
877 if (node < 0) /* curr_cpu was likely still -1 */
878 return 0;
879
880 node_present[node] = 1;
881 }
882
883 nodes = 0;
884
885 for (n = 0; n < MAX_NR_NODES; n++)
886 nodes += node_present[n];
887
888 return nodes;
889 }
890
891 /*
892 * Count the number of distinct process-threads a node contains.
893 *
894 * A count of 1 means that the node contains only a single
895 * process. If all nodes on the system contain at most one
896 * process then we are well-converged.
897 */
898 static int count_node_processes(int node)
899 {
900 int processes = 0;
901 int t, p;
902
903 for (p = 0; p < g->p.nr_proc; p++) {
904 for (t = 0; t < g->p.nr_threads; t++) {
905 struct thread_data *td;
906 int task_nr;
907 int n;
908
909 task_nr = p*g->p.nr_threads + t;
910 td = g->threads + task_nr;
911
912 n = numa_node_of_cpu(td->curr_cpu);
913 if (n == node) {
914 processes++;
915 break;
916 }
917 }
918 }
919
920 return processes;
921 }
922
923 static void calc_convergence_compression(int *strong)
924 {
925 unsigned int nodes_min, nodes_max;
926 int p;
927
928 nodes_min = -1;
929 nodes_max = 0;
930
931 for (p = 0; p < g->p.nr_proc; p++) {
932 unsigned int nodes = count_process_nodes(p);
933
934 if (!nodes) {
935 *strong = 0;
936 return;
937 }
938
939 nodes_min = min(nodes, nodes_min);
940 nodes_max = max(nodes, nodes_max);
941 }
942
943 /* Strong convergence: all threads compress on a single node: */
944 if (nodes_min == 1 && nodes_max == 1) {
945 *strong = 1;
946 } else {
947 *strong = 0;
948 tprintf(" {%d-%d}", nodes_min, nodes_max);
949 }
950 }
951
952 static void calc_convergence(double runtime_ns_max, double *convergence)
953 {
954 unsigned int loops_done_min, loops_done_max;
955 int process_groups;
956 int nodes[MAX_NR_NODES];
957 int distance;
958 int nr_min;
959 int nr_max;
960 int strong;
961 int sum;
962 int nr;
963 int node;
964 int cpu;
965 int t;
966
967 if (!g->p.show_convergence && !g->p.measure_convergence)
968 return;
969
970 for (node = 0; node < g->p.nr_nodes; node++)
971 nodes[node] = 0;
972
973 loops_done_min = -1;
974 loops_done_max = 0;
975
976 for (t = 0; t < g->p.nr_tasks; t++) {
977 struct thread_data *td = g->threads + t;
978 unsigned int loops_done;
979
980 cpu = td->curr_cpu;
981
982 /* Not all threads have written it yet: */
983 if (cpu < 0)
984 continue;
985
986 node = numa_node_of_cpu(cpu);
987
988 nodes[node]++;
989
990 loops_done = td->loops_done;
991 loops_done_min = min(loops_done, loops_done_min);
992 loops_done_max = max(loops_done, loops_done_max);
993 }
994
995 nr_max = 0;
996 nr_min = g->p.nr_tasks;
997 sum = 0;
998
999 for (node = 0; node < g->p.nr_nodes; node++) {
1000 if (!is_node_present(node))
1001 continue;
1002 nr = nodes[node];
1003 nr_min = min(nr, nr_min);
1004 nr_max = max(nr, nr_max);
1005 sum += nr;
1006 }
1007 BUG_ON(nr_min > nr_max);
1008
1009 BUG_ON(sum > g->p.nr_tasks);
1010
1011 if (0 && (sum < g->p.nr_tasks))
1012 return;
1013
1014 /*
1015 * Count the number of distinct process groups present
1016 * on nodes - when we are converged this will decrease
1017 * to g->p.nr_proc:
1018 */
1019 process_groups = 0;
1020
1021 for (node = 0; node < g->p.nr_nodes; node++) {
1022 int processes;
1023
1024 if (!is_node_present(node))
1025 continue;
1026 processes = count_node_processes(node);
1027 nr = nodes[node];
1028 tprintf(" %2d/%-2d", nr, processes);
1029
1030 process_groups += processes;
1031 }
1032
1033 distance = nr_max - nr_min;
1034
1035 tprintf(" [%2d/%-2d]", distance, process_groups);
1036
1037 tprintf(" l:%3d-%-3d (%3d)",
1038 loops_done_min, loops_done_max, loops_done_max-loops_done_min);
1039
1040 if (loops_done_min && loops_done_max) {
1041 double skew = 1.0 - (double)loops_done_min/loops_done_max;
1042
1043 tprintf(" [%4.1f%%]", skew * 100.0);
1044 }
1045
1046 calc_convergence_compression(&strong);
1047
1048 if (strong && process_groups == g->p.nr_proc) {
1049 if (!*convergence) {
1050 *convergence = runtime_ns_max;
1051 tprintf(" (%6.1fs converged)\n", *convergence/1e9);
1052 if (g->p.measure_convergence) {
1053 g->all_converged = true;
1054 g->stop_work = true;
1055 }
1056 }
1057 } else {
1058 if (*convergence) {
1059 tprintf(" (%6.1fs de-converged)", runtime_ns_max/1e9);
1060 *convergence = 0;
1061 }
1062 tprintf("\n");
1063 }
1064 }
1065
1066 static void show_summary(double runtime_ns_max, int l, double *convergence)
1067 {
1068 tprintf("\r # %5.1f%% [%.1f mins]",
1069 (double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max/1e9 / 60.0);
1070
1071 calc_convergence(runtime_ns_max, convergence);
1072
1073 if (g->p.show_details >= 0)
1074 fflush(stdout);
1075 }
1076
1077 static void *worker_thread(void *__tdata)
1078 {
1079 struct thread_data *td = __tdata;
1080 struct timeval start0, start, stop, diff;
1081 int process_nr = td->process_nr;
1082 int thread_nr = td->thread_nr;
1083 unsigned long last_perturbance;
1084 int task_nr = td->task_nr;
1085 int details = g->p.show_details;
1086 int first_task, last_task;
1087 double convergence = 0;
1088 u64 val = td->val;
1089 double runtime_ns_max;
1090 u8 *global_data;
1091 u8 *process_data;
1092 u8 *thread_data;
1093 u64 bytes_done;
1094 long work_done;
1095 u32 l;
1096 struct rusage rusage;
1097
1098 bind_to_cpumask(td->bind_cpumask);
1099 bind_to_memnode(td->bind_node);
1100
1101 set_taskname("thread %d/%d", process_nr, thread_nr);
1102
1103 global_data = g->data;
1104 process_data = td->process_data;
1105 thread_data = setup_private_data(g->p.bytes_thread);
1106
1107 bytes_done = 0;
1108
1109 last_task = 0;
1110 if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1)
1111 last_task = 1;
1112
1113 first_task = 0;
1114 if (process_nr == 0 && thread_nr == 0)
1115 first_task = 1;
1116
1117 if (details >= 2) {
1118 printf("# thread %2d / %2d global mem: %p, process mem: %p, thread mem: %p\n",
1119 process_nr, thread_nr, global_data, process_data, thread_data);
1120 }
1121
1122 if (g->p.serialize_startup) {
1123 pthread_mutex_lock(&g->startup_mutex);
1124 g->nr_tasks_started++;
1125 pthread_mutex_unlock(&g->startup_mutex);
1126
1127 /* Here we will wait for the main process to start us all at once: */
1128 pthread_mutex_lock(&g->start_work_mutex);
1129 g->nr_tasks_working++;
1130
1131 /* Last one wake the main process: */
1132 if (g->nr_tasks_working == g->p.nr_tasks)
1133 pthread_mutex_unlock(&g->startup_done_mutex);
1134
1135 pthread_mutex_unlock(&g->start_work_mutex);
1136 }
1137
1138 gettimeofday(&start0, NULL);
1139
1140 start = stop = start0;
1141 last_perturbance = start.tv_sec;
1142
1143 for (l = 0; l < g->p.nr_loops; l++) {
1144 start = stop;
1145
1146 if (g->stop_work)
1147 break;
1148
1149 val += do_work(global_data, g->p.bytes_global, process_nr, g->p.nr_proc, l, val);
1150 val += do_work(process_data, g->p.bytes_process, thread_nr, g->p.nr_threads, l, val);
1151 val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val);
1152
1153 if (g->p.sleep_usecs) {
1154 pthread_mutex_lock(td->process_lock);
1155 usleep(g->p.sleep_usecs);
1156 pthread_mutex_unlock(td->process_lock);
1157 }
1158 /*
1159 * Amount of work to be done under a process-global lock:
1160 */
1161 if (g->p.bytes_process_locked) {
1162 pthread_mutex_lock(td->process_lock);
1163 val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val);
1164 pthread_mutex_unlock(td->process_lock);
1165 }
1166
1167 work_done = g->p.bytes_global + g->p.bytes_process +
1168 g->p.bytes_process_locked + g->p.bytes_thread;
1169
1170 update_curr_cpu(task_nr, work_done);
1171 bytes_done += work_done;
1172
1173 if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs)
1174 continue;
1175
1176 td->loops_done = l;
1177
1178 gettimeofday(&stop, NULL);
1179
1180 /* Check whether our max runtime timed out: */
1181 if (g->p.nr_secs) {
1182 timersub(&stop, &start0, &diff);
1183 if ((u32)diff.tv_sec >= g->p.nr_secs) {
1184 g->stop_work = true;
1185 break;
1186 }
1187 }
1188
1189 /* Update the summary at most once per second: */
1190 if (start.tv_sec == stop.tv_sec)
1191 continue;
1192
1193 /*
1194 * Perturb the first task's equilibrium every g->p.perturb_secs seconds,
1195 * by migrating to CPU#0:
1196 */
1197 if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs) {
1198 cpu_set_t orig_mask;
1199 int target_cpu;
1200 int this_cpu;
1201
1202 last_perturbance = stop.tv_sec;
1203
1204 /*
1205 * Depending on where we are running, move into
1206 * the other half of the system, to create some
1207 * real disturbance:
1208 */
1209 this_cpu = g->threads[task_nr].curr_cpu;
1210 if (this_cpu < g->p.nr_cpus/2)
1211 target_cpu = g->p.nr_cpus-1;
1212 else
1213 target_cpu = 0;
1214
1215 orig_mask = bind_to_cpu(target_cpu);
1216
1217 /* Here we are running on the target CPU already */
1218 if (details >= 1)
1219 printf(" (injecting perturbalance, moved to CPU#%d)\n", target_cpu);
1220
1221 bind_to_cpumask(orig_mask);
1222 }
1223
1224 if (details >= 3) {
1225 timersub(&stop, &start, &diff);
1226 runtime_ns_max = diff.tv_sec * 1000000000;
1227 runtime_ns_max += diff.tv_usec * 1000;
1228
1229 if (details >= 0) {
1230 printf(" #%2d / %2d: %14.2lf nsecs/op [val: %016"PRIx64"]\n",
1231 process_nr, thread_nr, runtime_ns_max / bytes_done, val);
1232 }
1233 fflush(stdout);
1234 }
1235 if (!last_task)
1236 continue;
1237
1238 timersub(&stop, &start0, &diff);
1239 runtime_ns_max = diff.tv_sec * 1000000000ULL;
1240 runtime_ns_max += diff.tv_usec * 1000ULL;
1241
1242 show_summary(runtime_ns_max, l, &convergence);
1243 }
1244
1245 gettimeofday(&stop, NULL);
1246 timersub(&stop, &start0, &diff);
1247 td->runtime_ns = diff.tv_sec * 1000000000ULL;
1248 td->runtime_ns += diff.tv_usec * 1000ULL;
1249 td->speed_gbs = bytes_done / (td->runtime_ns / 1e9) / 1e9;
1250
1251 getrusage(RUSAGE_THREAD, &rusage);
1252 td->system_time_ns = rusage.ru_stime.tv_sec * 1000000000ULL;
1253 td->system_time_ns += rusage.ru_stime.tv_usec * 1000ULL;
1254 td->user_time_ns = rusage.ru_utime.tv_sec * 1000000000ULL;
1255 td->user_time_ns += rusage.ru_utime.tv_usec * 1000ULL;
1256
1257 free_data(thread_data, g->p.bytes_thread);
1258
1259 pthread_mutex_lock(&g->stop_work_mutex);
1260 g->bytes_done += bytes_done;
1261 pthread_mutex_unlock(&g->stop_work_mutex);
1262
1263 return NULL;
1264 }
1265
1266 /*
1267 * A worker process starts a couple of threads:
1268 */
1269 static void worker_process(int process_nr)
1270 {
1271 pthread_mutex_t process_lock;
1272 struct thread_data *td;
1273 pthread_t *pthreads;
1274 u8 *process_data;
1275 int task_nr;
1276 int ret;
1277 int t;
1278
1279 pthread_mutex_init(&process_lock, NULL);
1280 set_taskname("process %d", process_nr);
1281
1282 /*
1283 * Pick up the memory policy and the CPU binding of our first thread,
1284 * so that we initialize memory accordingly:
1285 */
1286 task_nr = process_nr*g->p.nr_threads;
1287 td = g->threads + task_nr;
1288
1289 bind_to_memnode(td->bind_node);
1290 bind_to_cpumask(td->bind_cpumask);
1291
1292 pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t));
1293 process_data = setup_private_data(g->p.bytes_process);
1294
1295 if (g->p.show_details >= 3) {
1296 printf(" # process %2d global mem: %p, process mem: %p\n",
1297 process_nr, g->data, process_data);
1298 }
1299
1300 for (t = 0; t < g->p.nr_threads; t++) {
1301 task_nr = process_nr*g->p.nr_threads + t;
1302 td = g->threads + task_nr;
1303
1304 td->process_data = process_data;
1305 td->process_nr = process_nr;
1306 td->thread_nr = t;
1307 td->task_nr = task_nr;
1308 td->val = rand();
1309 td->curr_cpu = -1;
1310 td->process_lock = &process_lock;
1311
1312 ret = pthread_create(pthreads + t, NULL, worker_thread, td);
1313 BUG_ON(ret);
1314 }
1315
1316 for (t = 0; t < g->p.nr_threads; t++) {
1317 ret = pthread_join(pthreads[t], NULL);
1318 BUG_ON(ret);
1319 }
1320
1321 free_data(process_data, g->p.bytes_process);
1322 free(pthreads);
1323 }
1324
1325 static void print_summary(void)
1326 {
1327 if (g->p.show_details < 0)
1328 return;
1329
1330 printf("\n ###\n");
1331 printf(" # %d %s will execute (on %d nodes, %d CPUs):\n",
1332 g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", nr_numa_nodes(), g->p.nr_cpus);
1333 printf(" # %5dx %5ldMB global shared mem operations\n",
1334 g->p.nr_loops, g->p.bytes_global/1024/1024);
1335 printf(" # %5dx %5ldMB process shared mem operations\n",
1336 g->p.nr_loops, g->p.bytes_process/1024/1024);
1337 printf(" # %5dx %5ldMB thread local mem operations\n",
1338 g->p.nr_loops, g->p.bytes_thread/1024/1024);
1339
1340 printf(" ###\n");
1341
1342 printf("\n ###\n"); fflush(stdout);
1343 }
1344
1345 static void init_thread_data(void)
1346 {
1347 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;
1348 int t;
1349
1350 g->threads = zalloc_shared_data(size);
1351
1352 for (t = 0; t < g->p.nr_tasks; t++) {
1353 struct thread_data *td = g->threads + t;
1354 int cpu;
1355
1356 /* Allow all nodes by default: */
1357 td->bind_node = -1;
1358
1359 /* Allow all CPUs by default: */
1360 CPU_ZERO(&td->bind_cpumask);
1361 for (cpu = 0; cpu < g->p.nr_cpus; cpu++)
1362 CPU_SET(cpu, &td->bind_cpumask);
1363 }
1364 }
1365
1366 static void deinit_thread_data(void)
1367 {
1368 ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;
1369
1370 free_data(g->threads, size);
1371 }
1372
1373 static int init(void)
1374 {
1375 g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0);
1376
1377 /* Copy over options: */
1378 g->p = p0;
1379
1380 g->p.nr_cpus = numa_num_configured_cpus();
1381
1382 g->p.nr_nodes = numa_max_node() + 1;
1383
1384 /* char array in count_process_nodes(): */
1385 BUG_ON(g->p.nr_nodes > MAX_NR_NODES || g->p.nr_nodes < 0);
1386
1387 if (g->p.show_quiet && !g->p.show_details)
1388 g->p.show_details = -1;
1389
1390 /* Some memory should be specified: */
1391 if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str)
1392 return -1;
1393
1394 if (g->p.mb_global_str) {
1395 g->p.mb_global = atof(g->p.mb_global_str);
1396 BUG_ON(g->p.mb_global < 0);
1397 }
1398
1399 if (g->p.mb_proc_str) {
1400 g->p.mb_proc = atof(g->p.mb_proc_str);
1401 BUG_ON(g->p.mb_proc < 0);
1402 }
1403
1404 if (g->p.mb_proc_locked_str) {
1405 g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str);
1406 BUG_ON(g->p.mb_proc_locked < 0);
1407 BUG_ON(g->p.mb_proc_locked > g->p.mb_proc);
1408 }
1409
1410 if (g->p.mb_thread_str) {
1411 g->p.mb_thread = atof(g->p.mb_thread_str);
1412 BUG_ON(g->p.mb_thread < 0);
1413 }
1414
1415 BUG_ON(g->p.nr_threads <= 0);
1416 BUG_ON(g->p.nr_proc <= 0);
1417
1418 g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads;
1419
1420 g->p.bytes_global = g->p.mb_global *1024L*1024L;
1421 g->p.bytes_process = g->p.mb_proc *1024L*1024L;
1422 g->p.bytes_process_locked = g->p.mb_proc_locked *1024L*1024L;
1423 g->p.bytes_thread = g->p.mb_thread *1024L*1024L;
1424
1425 g->data = setup_shared_data(g->p.bytes_global);
1426
1427 /* Startup serialization: */
1428 init_global_mutex(&g->start_work_mutex);
1429 init_global_mutex(&g->startup_mutex);
1430 init_global_mutex(&g->startup_done_mutex);
1431 init_global_mutex(&g->stop_work_mutex);
1432
1433 init_thread_data();
1434
1435 tprintf("#\n");
1436 if (parse_setup_cpu_list() || parse_setup_node_list())
1437 return -1;
1438 tprintf("#\n");
1439
1440 print_summary();
1441
1442 return 0;
1443 }
1444
1445 static void deinit(void)
1446 {
1447 free_data(g->data, g->p.bytes_global);
1448 g->data = NULL;
1449
1450 deinit_thread_data();
1451
1452 free_data(g, sizeof(*g));
1453 g = NULL;
1454 }
1455
1456 /*
1457 * Print a short or long result, depending on the verbosity setting:
1458 */
1459 static void print_res(const char *name, double val,
1460 const char *txt_unit, const char *txt_short, const char *txt_long)
1461 {
1462 if (!name)
1463 name = "main,";
1464
1465 if (!g->p.show_quiet)
1466 printf(" %-30s %15.3f, %-15s %s\n", name, val, txt_unit, txt_short);
1467 else
1468 printf(" %14.3f %s\n", val, txt_long);
1469 }
1470
1471 static int __bench_numa(const char *name)
1472 {
1473 struct timeval start, stop, diff;
1474 u64 runtime_ns_min, runtime_ns_sum;
1475 pid_t *pids, pid, wpid;
1476 double delta_runtime;
1477 double runtime_avg;
1478 double runtime_sec_max;
1479 double runtime_sec_min;
1480 int wait_stat;
1481 double bytes;
1482 int i, t, p;
1483
1484 if (init())
1485 return -1;
1486
1487 pids = zalloc(g->p.nr_proc * sizeof(*pids));
1488 pid = -1;
1489
1490 /* All threads try to acquire it, this way we can wait for them to start up: */
1491 pthread_mutex_lock(&g->start_work_mutex);
1492
1493 if (g->p.serialize_startup) {
1494 tprintf(" #\n");
1495 tprintf(" # Startup synchronization: ..."); fflush(stdout);
1496 }
1497
1498 gettimeofday(&start, NULL);
1499
1500 for (i = 0; i < g->p.nr_proc; i++) {
1501 pid = fork();
1502 dprintf(" # process %2d: PID %d\n", i, pid);
1503
1504 BUG_ON(pid < 0);
1505 if (!pid) {
1506 /* Child process: */
1507 worker_process(i);
1508
1509 exit(0);
1510 }
1511 pids[i] = pid;
1512
1513 }
1514 /* Wait for all the threads to start up: */
1515 while (g->nr_tasks_started != g->p.nr_tasks)
1516 usleep(1000);
1517
1518 BUG_ON(g->nr_tasks_started != g->p.nr_tasks);
1519
1520 if (g->p.serialize_startup) {
1521 double startup_sec;
1522
1523 pthread_mutex_lock(&g->startup_done_mutex);
1524
1525 /* This will start all threads: */
1526 pthread_mutex_unlock(&g->start_work_mutex);
1527
1528 /* This mutex is locked - the last started thread will wake us: */
1529 pthread_mutex_lock(&g->startup_done_mutex);
1530
1531 gettimeofday(&stop, NULL);
1532
1533 timersub(&stop, &start, &diff);
1534
1535 startup_sec = diff.tv_sec * 1000000000.0;
1536 startup_sec += diff.tv_usec * 1000.0;
1537 startup_sec /= 1e9;
1538
1539 tprintf(" threads initialized in %.6f seconds.\n", startup_sec);
1540 tprintf(" #\n");
1541
1542 start = stop;
1543 pthread_mutex_unlock(&g->startup_done_mutex);
1544 } else {
1545 gettimeofday(&start, NULL);
1546 }
1547
1548 /* Parent process: */
1549
1550
1551 for (i = 0; i < g->p.nr_proc; i++) {
1552 wpid = waitpid(pids[i], &wait_stat, 0);
1553 BUG_ON(wpid < 0);
1554 BUG_ON(!WIFEXITED(wait_stat));
1555
1556 }
1557
1558 runtime_ns_sum = 0;
1559 runtime_ns_min = -1LL;
1560
1561 for (t = 0; t < g->p.nr_tasks; t++) {
1562 u64 thread_runtime_ns = g->threads[t].runtime_ns;
1563
1564 runtime_ns_sum += thread_runtime_ns;
1565 runtime_ns_min = min(thread_runtime_ns, runtime_ns_min);
1566 }
1567
1568 gettimeofday(&stop, NULL);
1569 timersub(&stop, &start, &diff);
1570
1571 BUG_ON(bench_format != BENCH_FORMAT_DEFAULT);
1572
1573 tprintf("\n ###\n");
1574 tprintf("\n");
1575
1576 runtime_sec_max = diff.tv_sec * 1000000000.0;
1577 runtime_sec_max += diff.tv_usec * 1000.0;
1578 runtime_sec_max /= 1e9;
1579
1580 runtime_sec_min = runtime_ns_min/1e9;
1581
1582 bytes = g->bytes_done;
1583 runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / 1e9;
1584
1585 if (g->p.measure_convergence) {
1586 print_res(name, runtime_sec_max,
1587 "secs,", "NUMA-convergence-latency", "secs latency to NUMA-converge");
1588 }
1589
1590 print_res(name, runtime_sec_max,
1591 "secs,", "runtime-max/thread", "secs slowest (max) thread-runtime");
1592
1593 print_res(name, runtime_sec_min,
1594 "secs,", "runtime-min/thread", "secs fastest (min) thread-runtime");
1595
1596 print_res(name, runtime_avg,
1597 "secs,", "runtime-avg/thread", "secs average thread-runtime");
1598
1599 delta_runtime = (runtime_sec_max - runtime_sec_min)/2.0;
1600 print_res(name, delta_runtime / runtime_sec_max * 100.0,
1601 "%,", "spread-runtime/thread", "% difference between max/avg runtime");
1602
1603 print_res(name, bytes / g->p.nr_tasks / 1e9,
1604 "GB,", "data/thread", "GB data processed, per thread");
1605
1606 print_res(name, bytes / 1e9,
1607 "GB,", "data-total", "GB data processed, total");
1608
1609 print_res(name, runtime_sec_max * 1e9 / (bytes / g->p.nr_tasks),
1610 "nsecs,", "runtime/byte/thread","nsecs/byte/thread runtime");
1611
1612 print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max,
1613 "GB/sec,", "thread-speed", "GB/sec/thread speed");
1614
1615 print_res(name, bytes / runtime_sec_max / 1e9,
1616 "GB/sec,", "total-speed", "GB/sec total speed");
1617
1618 if (g->p.show_details >= 2) {
1619 char tname[14 + 2 * 10 + 1];
1620 struct thread_data *td;
1621 for (p = 0; p < g->p.nr_proc; p++) {
1622 for (t = 0; t < g->p.nr_threads; t++) {
1623 memset(tname, 0, sizeof(tname));
1624 td = g->threads + p*g->p.nr_threads + t;
1625 snprintf(tname, sizeof(tname), "process%d:thread%d", p, t);
1626 print_res(tname, td->speed_gbs,
1627 "GB/sec", "thread-speed", "GB/sec/thread speed");
1628 print_res(tname, td->system_time_ns / 1e9,
1629 "secs", "thread-system-time", "system CPU time/thread");
1630 print_res(tname, td->user_time_ns / 1e9,
1631 "secs", "thread-user-time", "user CPU time/thread");
1632 }
1633 }
1634 }
1635
1636 free(pids);
1637
1638 deinit();
1639
1640 return 0;
1641 }
1642
1643 #define MAX_ARGS 50
1644
1645 static int command_size(const char **argv)
1646 {
1647 int size = 0;
1648
1649 while (*argv) {
1650 size++;
1651 argv++;
1652 }
1653
1654 BUG_ON(size >= MAX_ARGS);
1655
1656 return size;
1657 }
1658
1659 static void init_params(struct params *p, const char *name, int argc, const char **argv)
1660 {
1661 int i;
1662
1663 printf("\n # Running %s \"perf bench numa", name);
1664
1665 for (i = 0; i < argc; i++)
1666 printf(" %s", argv[i]);
1667
1668 printf("\"\n");
1669
1670 memset(p, 0, sizeof(*p));
1671
1672 /* Initialize nonzero defaults: */
1673
1674 p->serialize_startup = 1;
1675 p->data_reads = true;
1676 p->data_writes = true;
1677 p->data_backwards = true;
1678 p->data_rand_walk = true;
1679 p->nr_loops = -1;
1680 p->init_random = true;
1681 p->mb_global_str = "1";
1682 p->nr_proc = 1;
1683 p->nr_threads = 1;
1684 p->nr_secs = 5;
1685 p->run_all = argc == 1;
1686 }
1687
1688 static int run_bench_numa(const char *name, const char **argv)
1689 {
1690 int argc = command_size(argv);
1691
1692 init_params(&p0, name, argc, argv);
1693 argc = parse_options(argc, argv, options, bench_numa_usage, 0);
1694 if (argc)
1695 goto err;
1696
1697 if (__bench_numa(name))
1698 goto err;
1699
1700 return 0;
1701
1702 err:
1703 return -1;
1704 }
1705
1706 #define OPT_BW_RAM "-s", "20", "-zZq", "--thp", " 1", "--no-data_rand_walk"
1707 #define OPT_BW_RAM_NOTHP OPT_BW_RAM, "--thp", "-1"
1708
1709 #define OPT_CONV "-s", "100", "-zZ0qcm", "--thp", " 1"
1710 #define OPT_CONV_NOTHP OPT_CONV, "--thp", "-1"
1711
1712 #define OPT_BW "-s", "20", "-zZ0q", "--thp", " 1"
1713 #define OPT_BW_NOTHP OPT_BW, "--thp", "-1"
1714
1715 /*
1716 * The built-in test-suite executed by "perf bench numa -a".
1717 *
1718 * (A minimum of 4 nodes and 16 GB of RAM is recommended.)
1719 */
1720 static const char *tests[][MAX_ARGS] = {
1721 /* Basic single-stream NUMA bandwidth measurements: */
1722 { "RAM-bw-local,", "mem", "-p", "1", "-t", "1", "-P", "1024",
1723 "-C" , "0", "-M", "0", OPT_BW_RAM },
1724 { "RAM-bw-local-NOTHP,",
1725 "mem", "-p", "1", "-t", "1", "-P", "1024",
1726 "-C" , "0", "-M", "0", OPT_BW_RAM_NOTHP },
1727 { "RAM-bw-remote,", "mem", "-p", "1", "-t", "1", "-P", "1024",
1728 "-C" , "0", "-M", "1", OPT_BW_RAM },
1729
1730 /* 2-stream NUMA bandwidth measurements: */
1731 { "RAM-bw-local-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1732 "-C", "0,2", "-M", "0x2", OPT_BW_RAM },
1733 { "RAM-bw-remote-2x,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1734 "-C", "0,2", "-M", "1x2", OPT_BW_RAM },
1735
1736 /* Cross-stream NUMA bandwidth measurement: */
1737 { "RAM-bw-cross,", "mem", "-p", "2", "-t", "1", "-P", "1024",
1738 "-C", "0,8", "-M", "1,0", OPT_BW_RAM },
1739
1740 /* Convergence latency measurements: */
1741 { " 1x3-convergence,", "mem", "-p", "1", "-t", "3", "-P", "512", OPT_CONV },
1742 { " 1x4-convergence,", "mem", "-p", "1", "-t", "4", "-P", "512", OPT_CONV },
1743 { " 1x6-convergence,", "mem", "-p", "1", "-t", "6", "-P", "1020", OPT_CONV },
1744 { " 2x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV },
1745 { " 3x3-convergence,", "mem", "-p", "3", "-t", "3", "-P", "1020", OPT_CONV },
1746 { " 4x4-convergence,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV },
1747 { " 4x4-convergence-NOTHP,",
1748 "mem", "-p", "4", "-t", "4", "-P", "512", OPT_CONV_NOTHP },
1749 { " 4x6-convergence,", "mem", "-p", "4", "-t", "6", "-P", "1020", OPT_CONV },
1750 { " 4x8-convergence,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_CONV },
1751 { " 8x4-convergence,", "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV },
1752 { " 8x4-convergence-NOTHP,",
1753 "mem", "-p", "8", "-t", "4", "-P", "512", OPT_CONV_NOTHP },
1754 { " 3x1-convergence,", "mem", "-p", "3", "-t", "1", "-P", "512", OPT_CONV },
1755 { " 4x1-convergence,", "mem", "-p", "4", "-t", "1", "-P", "512", OPT_CONV },
1756 { " 8x1-convergence,", "mem", "-p", "8", "-t", "1", "-P", "512", OPT_CONV },
1757 { "16x1-convergence,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_CONV },
1758 { "32x1-convergence,", "mem", "-p", "32", "-t", "1", "-P", "128", OPT_CONV },
1759
1760 /* Various NUMA process/thread layout bandwidth measurements: */
1761 { " 2x1-bw-process,", "mem", "-p", "2", "-t", "1", "-P", "1024", OPT_BW },
1762 { " 3x1-bw-process,", "mem", "-p", "3", "-t", "1", "-P", "1024", OPT_BW },
1763 { " 4x1-bw-process,", "mem", "-p", "4", "-t", "1", "-P", "1024", OPT_BW },
1764 { " 8x1-bw-process,", "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW },
1765 { " 8x1-bw-process-NOTHP,",
1766 "mem", "-p", "8", "-t", "1", "-P", " 512", OPT_BW_NOTHP },
1767 { "16x1-bw-process,", "mem", "-p", "16", "-t", "1", "-P", "256", OPT_BW },
1768
1769 { " 4x1-bw-thread,", "mem", "-p", "1", "-t", "4", "-T", "256", OPT_BW },
1770 { " 8x1-bw-thread,", "mem", "-p", "1", "-t", "8", "-T", "256", OPT_BW },
1771 { "16x1-bw-thread,", "mem", "-p", "1", "-t", "16", "-T", "128", OPT_BW },
1772 { "32x1-bw-thread,", "mem", "-p", "1", "-t", "32", "-T", "64", OPT_BW },
1773
1774 { " 2x3-bw-thread,", "mem", "-p", "2", "-t", "3", "-P", "512", OPT_BW },
1775 { " 4x4-bw-thread,", "mem", "-p", "4", "-t", "4", "-P", "512", OPT_BW },
1776 { " 4x6-bw-thread,", "mem", "-p", "4", "-t", "6", "-P", "512", OPT_BW },
1777 { " 4x8-bw-thread,", "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW },
1778 { " 4x8-bw-thread-NOTHP,",
1779 "mem", "-p", "4", "-t", "8", "-P", "512", OPT_BW_NOTHP },
1780 { " 3x3-bw-thread,", "mem", "-p", "3", "-t", "3", "-P", "512", OPT_BW },
1781 { " 5x5-bw-thread,", "mem", "-p", "5", "-t", "5", "-P", "512", OPT_BW },
1782
1783 { "2x16-bw-thread,", "mem", "-p", "2", "-t", "16", "-P", "512", OPT_BW },
1784 { "1x32-bw-thread,", "mem", "-p", "1", "-t", "32", "-P", "2048", OPT_BW },
1785
1786 { "numa02-bw,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW },
1787 { "numa02-bw-NOTHP,", "mem", "-p", "1", "-t", "32", "-T", "32", OPT_BW_NOTHP },
1788 { "numa01-bw-thread,", "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW },
1789 { "numa01-bw-thread-NOTHP,",
1790 "mem", "-p", "2", "-t", "16", "-T", "192", OPT_BW_NOTHP },
1791 };
1792
1793 static int bench_all(void)
1794 {
1795 int nr = ARRAY_SIZE(tests);
1796 int ret;
1797 int i;
1798
1799 ret = system("echo ' #'; echo ' # Running test on: '$(uname -a); echo ' #'");
1800 BUG_ON(ret < 0);
1801
1802 for (i = 0; i < nr; i++) {
1803 run_bench_numa(tests[i][0], tests[i] + 1);
1804 }
1805
1806 printf("\n");
1807
1808 return 0;
1809 }
1810
1811 int bench_numa(int argc, const char **argv, const char *prefix __maybe_unused)
1812 {
1813 init_params(&p0, "main,", argc, argv);
1814 argc = parse_options(argc, argv, options, bench_numa_usage, 0);
1815 if (argc)
1816 goto err;
1817
1818 if (p0.run_all)
1819 return bench_all();
1820
1821 if (__bench_numa(NULL))
1822 goto err;
1823
1824 return 0;
1825
1826 err:
1827 usage_with_options(numa_usage, options);
1828 return -1;
1829 }
Linux with added FPC-III support