tools/perf/util/bpf_counter.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* Copyright (c) 2019 Facebook */
   4
   5 #include <assert.h>
   6 #include <limits.h>
   7 #include <unistd.h>
   8 #include <sys/file.h>
   9 #include <sys/time.h>
  10 #include <linux/err.h>
  11 #include <linux/zalloc.h>
  12 #include <api/fs/fs.h>
  13 #include <perf/bpf_perf.h>
  14
  15 #include "bpf_counter.h"
  16 #include "bpf-utils.h"
  17 #include "counts.h"
  18 #include "debug.h"
  19 #include "evsel.h"
  20 #include "evlist.h"
  21 #include "target.h"
  22 #include "cgroup.h"
  23 #include "cpumap.h"
  24 #include "thread_map.h"
  25
  26 #include "bpf_skel/bpf_prog_profiler.skel.h"
  27 #include "bpf_skel/bperf_u.h"
  28 #include "bpf_skel/bperf_leader.skel.h"
  29 #include "bpf_skel/bperf_follower.skel.h"
  30
  31 #define ATTR_MAP_SIZE 16
  32
  33 static inline void *u64_to_ptr(__u64 ptr)
  34 {
  35         return (void *)(unsigned long)ptr;
  36 }
  37
  38 static struct bpf_counter *bpf_counter_alloc(void)
  39 {
  40         struct bpf_counter *counter;
  41
  42         counter = zalloc(sizeof(*counter));
  43         if (counter)
  44                 INIT_LIST_HEAD(&counter->list);
  45         return counter;
  46 }
  47
  48 static int bpf_program_profiler__destroy(struct evsel *evsel)
  49 {
  50         struct bpf_counter *counter, *tmp;
  51
  52         list_for_each_entry_safe(counter, tmp,
  53                                  &evsel->bpf_counter_list, list) {
  54                 list_del_init(&counter->list);
  55                 bpf_prog_profiler_bpf__destroy(counter->skel);
  56                 free(counter);
  57         }
  58         assert(list_empty(&evsel->bpf_counter_list));
  59
  60         return 0;
  61 }
  62
  63 static char *bpf_target_prog_name(int tgt_fd)
  64 {
  65         struct bpf_func_info *func_info;
  66         struct perf_bpil *info_linear;
  67         const struct btf_type *t;
  68         struct btf *btf = NULL;
  69         char *name = NULL;
  70
  71         info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO);
  72         if (IS_ERR_OR_NULL(info_linear)) {
  73                 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd);
  74                 return NULL;
  75         }
  76
  77         if (info_linear->info.btf_id == 0) {
  78                 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd);
  79                 goto out;
  80         }
  81
  82         btf = btf__load_from_kernel_by_id(info_linear->info.btf_id);
  83         if (libbpf_get_error(btf)) {
  84                 pr_debug("failed to load btf for prog FD %d\n", tgt_fd);
  85                 goto out;
  86         }
  87
  88         func_info = u64_to_ptr(info_linear->info.func_info);
  89         t = btf__type_by_id(btf, func_info[0].type_id);
  90         if (!t) {
  91                 pr_debug("btf %d doesn't have type %d\n",
  92                          info_linear->info.btf_id, func_info[0].type_id);
  93                 goto out;
  94         }
  95         name = strdup(btf__name_by_offset(btf, t->name_off));
  96 out:
  97         btf__free(btf);
  98         free(info_linear);
  99         return name;
 100 }
 101
 102 static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id)
 103 {
 104         struct bpf_prog_profiler_bpf *skel;
 105         struct bpf_counter *counter;
 106         struct bpf_program *prog;
 107         char *prog_name = NULL;
 108         int prog_fd;
 109         int err;
 110
 111         prog_fd = bpf_prog_get_fd_by_id(prog_id);
 112         if (prog_fd < 0) {
 113                 pr_err("Failed to open fd for bpf prog %u\n", prog_id);
 114                 return -1;
 115         }
 116         counter = bpf_counter_alloc();
 117         if (!counter) {
 118                 close(prog_fd);
 119                 return -1;
 120         }
 121
 122         skel = bpf_prog_profiler_bpf__open();
 123         if (!skel) {
 124                 pr_err("Failed to open bpf skeleton\n");
 125                 goto err_out;
 126         }
 127
 128         skel->rodata->num_cpu = evsel__nr_cpus(evsel);
 129
 130         bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel));
 131         bpf_map__set_max_entries(skel->maps.fentry_readings, 1);
 132         bpf_map__set_max_entries(skel->maps.accum_readings, 1);
 133
 134         prog_name = bpf_target_prog_name(prog_fd);
 135         if (!prog_name) {
 136                 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id);
 137                 goto err_out;
 138         }
 139
 140         bpf_object__for_each_program(prog, skel->obj) {
 141                 err = bpf_program__set_attach_target(prog, prog_fd, prog_name);
 142                 if (err) {
 143                         pr_err("bpf_program__set_attach_target failed.\n"
 144                                "Does bpf prog %u have BTF?\n", prog_id);
 145                         goto err_out;
 146                 }
 147         }
 148         set_max_rlimit();
 149         err = bpf_prog_profiler_bpf__load(skel);
 150         if (err) {
 151                 pr_err("bpf_prog_profiler_bpf__load failed\n");
 152                 goto err_out;
 153         }
 154
 155         assert(skel != NULL);
 156         counter->skel = skel;
 157         list_add(&counter->list, &evsel->bpf_counter_list);
 158         free(prog_name);
 159         close(prog_fd);
 160         return 0;
 161 err_out:
 162         bpf_prog_profiler_bpf__destroy(skel);
 163         free(prog_name);
 164         free(counter);
 165         close(prog_fd);
 166         return -1;
 167 }
 168
 169 static int bpf_program_profiler__load(struct evsel *evsel, struct target *target)
 170 {
 171         char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p;
 172         u32 prog_id;
 173         int ret;
 174
 175         bpf_str_ = bpf_str = strdup(target->bpf_str);
 176         if (!bpf_str)
 177                 return -1;
 178
 179         while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) {
 180                 prog_id = strtoul(tok, &p, 10);
 181                 if (prog_id == 0 || prog_id == UINT_MAX ||
 182                     (*p != '\0' && *p != ',')) {
 183                         pr_err("Failed to parse bpf prog ids %s\n",
 184                                target->bpf_str);
 185                         free(bpf_str_);
 186                         return -1;
 187                 }
 188
 189                 ret = bpf_program_profiler_load_one(evsel, prog_id);
 190                 if (ret) {
 191                         bpf_program_profiler__destroy(evsel);
 192                         free(bpf_str_);
 193                         return -1;
 194                 }
 195                 bpf_str = NULL;
 196         }
 197         free(bpf_str_);
 198         return 0;
 199 }
 200
 201 static int bpf_program_profiler__enable(struct evsel *evsel)
 202 {
 203         struct bpf_counter *counter;
 204         int ret;
 205
 206         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 207                 assert(counter->skel != NULL);
 208                 ret = bpf_prog_profiler_bpf__attach(counter->skel);
 209                 if (ret) {
 210                         bpf_program_profiler__destroy(evsel);
 211                         return ret;
 212                 }
 213         }
 214         return 0;
 215 }
 216
 217 static int bpf_program_profiler__disable(struct evsel *evsel)
 218 {
 219         struct bpf_counter *counter;
 220
 221         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 222                 assert(counter->skel != NULL);
 223                 bpf_prog_profiler_bpf__detach(counter->skel);
 224         }
 225         return 0;
 226 }
 227
 228 static int bpf_program_profiler__read(struct evsel *evsel)
 229 {
 230         // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible
 231         // Sometimes possible > online, like on a Ryzen 3900X that has 24
 232         // threads but its possible showed 0-31 -acme
 233         int num_cpu_bpf = libbpf_num_possible_cpus();
 234         struct bpf_perf_event_value values[num_cpu_bpf];
 235         struct bpf_counter *counter;
 236         struct perf_counts_values *counts;
 237         int reading_map_fd;
 238         __u32 key = 0;
 239         int err, idx, bpf_cpu;
 240
 241         if (list_empty(&evsel->bpf_counter_list))
 242                 return -EAGAIN;
 243
 244         perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) {
 245                 counts = perf_counts(evsel->counts, idx, 0);
 246                 counts->val = 0;
 247                 counts->ena = 0;
 248                 counts->run = 0;
 249         }
 250         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 251                 struct bpf_prog_profiler_bpf *skel = counter->skel;
 252
 253                 assert(skel != NULL);
 254                 reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
 255
 256                 err = bpf_map_lookup_elem(reading_map_fd, &key, values);
 257                 if (err) {
 258                         pr_err("failed to read value\n");
 259                         return err;
 260                 }
 261
 262                 for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) {
 263                         idx = perf_cpu_map__idx(evsel__cpus(evsel),
 264                                                 (struct perf_cpu){.cpu = bpf_cpu});
 265                         if (idx == -1)
 266                                 continue;
 267                         counts = perf_counts(evsel->counts, idx, 0);
 268                         counts->val += values[bpf_cpu].counter;
 269                         counts->ena += values[bpf_cpu].enabled;
 270                         counts->run += values[bpf_cpu].running;
 271                 }
 272         }
 273         return 0;
 274 }
 275
 276 static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx,
 277                                             int fd)
 278 {
 279         struct bpf_prog_profiler_bpf *skel;
 280         struct bpf_counter *counter;
 281         int ret;
 282
 283         list_for_each_entry(counter, &evsel->bpf_counter_list, list) {
 284                 skel = counter->skel;
 285                 assert(skel != NULL);
 286
 287                 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events),
 288                                           &cpu_map_idx, &fd, BPF_ANY);
 289                 if (ret)
 290                         return ret;
 291         }
 292         return 0;
 293 }
 294
 295 struct bpf_counter_ops bpf_program_profiler_ops = {
 296         .load       = bpf_program_profiler__load,
 297         .enable     = bpf_program_profiler__enable,
 298         .disable    = bpf_program_profiler__disable,
 299         .read       = bpf_program_profiler__read,
 300         .destroy    = bpf_program_profiler__destroy,
 301         .install_pe = bpf_program_profiler__install_pe,
 302 };
 303
 304 static bool bperf_attr_map_compatible(int attr_map_fd)
 305 {
 306         struct bpf_map_info map_info = {0};
 307         __u32 map_info_len = sizeof(map_info);
 308         int err;
 309
 310         err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len);
 311
 312         if (err)
 313                 return false;
 314         return (map_info.key_size == sizeof(struct perf_event_attr)) &&
 315                 (map_info.value_size == sizeof(struct perf_event_attr_map_entry));
 316 }
 317
 318 static int bperf_lock_attr_map(struct target *target)
 319 {
 320         char path[PATH_MAX];
 321         int map_fd, err;
 322
 323         if (target->attr_map) {
 324                 scnprintf(path, PATH_MAX, "%s", target->attr_map);
 325         } else {
 326                 scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(),
 327                           BPF_PERF_DEFAULT_ATTR_MAP_PATH);
 328         }
 329
 330         if (access(path, F_OK)) {
 331                 map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL,
 332                                         sizeof(struct perf_event_attr),
 333                                         sizeof(struct perf_event_attr_map_entry),
 334                                         ATTR_MAP_SIZE, NULL);
 335                 if (map_fd < 0)
 336                         return -1;
 337
 338                 err = bpf_obj_pin(map_fd, path);
 339                 if (err) {
 340                         /* someone pinned the map in parallel? */
 341                         close(map_fd);
 342                         map_fd = bpf_obj_get(path);
 343                         if (map_fd < 0)
 344                                 return -1;
 345                 }
 346         } else {
 347                 map_fd = bpf_obj_get(path);
 348                 if (map_fd < 0)
 349                         return -1;
 350         }
 351
 352         if (!bperf_attr_map_compatible(map_fd)) {
 353                 close(map_fd);
 354                 return -1;
 355
 356         }
 357         err = flock(map_fd, LOCK_EX);
 358         if (err) {
 359                 close(map_fd);
 360                 return -1;
 361         }
 362         return map_fd;
 363 }
 364
 365 static int bperf_check_target(struct evsel *evsel,
 366                               struct target *target,
 367                               enum bperf_filter_type *filter_type,
 368                               __u32 *filter_entry_cnt)
 369 {
 370         if (evsel->core.leader->nr_members > 1) {
 371                 pr_err("bpf managed perf events do not yet support groups.\n");
 372                 return -1;
 373         }
 374
 375         /* determine filter type based on target */
 376         if (target->system_wide) {
 377                 *filter_type = BPERF_FILTER_GLOBAL;
 378                 *filter_entry_cnt = 1;
 379         } else if (target->cpu_list) {
 380                 *filter_type = BPERF_FILTER_CPU;
 381                 *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel));
 382         } else if (target->tid) {
 383                 *filter_type = BPERF_FILTER_PID;
 384                 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
 385         } else if (target->pid || evsel->evlist->workload.pid != -1) {
 386                 *filter_type = BPERF_FILTER_TGID;
 387                 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads);
 388         } else {
 389                 pr_err("bpf managed perf events do not yet support these targets.\n");
 390                 return -1;
 391         }
 392
 393         return 0;
 394 }
 395
 396 static  struct perf_cpu_map *all_cpu_map;
 397 static __u32 filter_entry_cnt;
 398
 399 static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd,
 400                                        struct perf_event_attr_map_entry *entry)
 401 {
 402         struct bperf_leader_bpf *skel = bperf_leader_bpf__open();
 403         int link_fd, diff_map_fd, err;
 404         struct bpf_link *link = NULL;
 405
 406         if (!skel) {
 407                 pr_err("Failed to open leader skeleton\n");
 408                 return -1;
 409         }
 410
 411         bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus());
 412         err = bperf_leader_bpf__load(skel);
 413         if (err) {
 414                 pr_err("Failed to load leader skeleton\n");
 415                 goto out;
 416         }
 417
 418         link = bpf_program__attach(skel->progs.on_switch);
 419         if (IS_ERR(link)) {
 420                 pr_err("Failed to attach leader program\n");
 421                 err = PTR_ERR(link);
 422                 goto out;
 423         }
 424
 425         link_fd = bpf_link__fd(link);
 426         diff_map_fd = bpf_map__fd(skel->maps.diff_readings);
 427         entry->link_id = bpf_link_get_id(link_fd);
 428         entry->diff_map_id = bpf_map_get_id(diff_map_fd);
 429         err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY);
 430         assert(err == 0);
 431
 432         evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id);
 433         assert(evsel->bperf_leader_link_fd >= 0);
 434
 435         /*
 436          * save leader_skel for install_pe, which is called within
 437          * following evsel__open_per_cpu call
 438          */
 439         evsel->leader_skel = skel;
 440         evsel__open_per_cpu(evsel, all_cpu_map, -1);
 441
 442 out:
 443         bperf_leader_bpf__destroy(skel);
 444         bpf_link__destroy(link);
 445         return err;
 446 }
 447
 448 static int bperf_attach_follower_program(struct bperf_follower_bpf *skel,
 449                                          enum bperf_filter_type filter_type,
 450                                          bool inherit)
 451 {
 452         struct bpf_link *link;
 453         int err = 0;
 454
 455         if ((filter_type == BPERF_FILTER_PID ||
 456             filter_type == BPERF_FILTER_TGID) && inherit)
 457                 /* attach all follower bpf progs to enable event inheritance */
 458                 err = bperf_follower_bpf__attach(skel);
 459         else {
 460                 link = bpf_program__attach(skel->progs.fexit_XXX);
 461                 if (IS_ERR(link))
 462                         err = PTR_ERR(link);
 463         }
 464
 465         return err;
 466 }
 467
 468 static int bperf__load(struct evsel *evsel, struct target *target)
 469 {
 470         struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff};
 471         int attr_map_fd, diff_map_fd = -1, err;
 472         enum bperf_filter_type filter_type;
 473         __u32 i;
 474
 475         if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt))
 476                 return -1;
 477
 478         if (!all_cpu_map) {
 479                 all_cpu_map = perf_cpu_map__new_online_cpus();
 480                 if (!all_cpu_map)
 481                         return -1;
 482         }
 483
 484         evsel->bperf_leader_prog_fd = -1;
 485         evsel->bperf_leader_link_fd = -1;
 486
 487         /*
 488          * Step 1: hold a fd on the leader program and the bpf_link, if
 489          * the program is not already gone, reload the program.
 490          * Use flock() to ensure exclusive access to the perf_event_attr
 491          * map.
 492          */
 493         attr_map_fd = bperf_lock_attr_map(target);
 494         if (attr_map_fd < 0) {
 495                 pr_err("Failed to lock perf_event_attr map\n");
 496                 return -1;
 497         }
 498
 499         err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry);
 500         if (err) {
 501                 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY);
 502                 if (err)
 503                         goto out;
 504         }
 505
 506         evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id);
 507         if (evsel->bperf_leader_link_fd < 0 &&
 508             bperf_reload_leader_program(evsel, attr_map_fd, &entry)) {
 509                 err = -1;
 510                 goto out;
 511         }
 512         /*
 513          * The bpf_link holds reference to the leader program, and the
 514          * leader program holds reference to the maps. Therefore, if
 515          * link_id is valid, diff_map_id should also be valid.
 516          */
 517         evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id(
 518                 bpf_link_get_prog_id(evsel->bperf_leader_link_fd));
 519         assert(evsel->bperf_leader_prog_fd >= 0);
 520
 521         diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id);
 522         assert(diff_map_fd >= 0);
 523
 524         /*
 525          * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check
 526          * whether the kernel support it
 527          */
 528         err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0);
 529         if (err) {
 530                 pr_err("The kernel does not support test_run for raw_tp BPF programs.\n"
 531                        "Therefore, --use-bpf might show inaccurate readings\n");
 532                 goto out;
 533         }
 534
 535         /* Step 2: load the follower skeleton */
 536         evsel->follower_skel = bperf_follower_bpf__open();
 537         if (!evsel->follower_skel) {
 538                 err = -1;
 539                 pr_err("Failed to open follower skeleton\n");
 540                 goto out;
 541         }
 542
 543         /* attach fexit program to the leader program */
 544         bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX,
 545                                        evsel->bperf_leader_prog_fd, "on_switch");
 546
 547         /* connect to leader diff_reading map */
 548         bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd);
 549
 550         /* set up reading map */
 551         bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings,
 552                                  filter_entry_cnt);
 553         err = bperf_follower_bpf__load(evsel->follower_skel);
 554         if (err) {
 555                 pr_err("Failed to load follower skeleton\n");
 556                 bperf_follower_bpf__destroy(evsel->follower_skel);
 557                 evsel->follower_skel = NULL;
 558                 goto out;
 559         }
 560
 561         for (i = 0; i < filter_entry_cnt; i++) {
 562                 int filter_map_fd;
 563                 __u32 key;
 564                 struct bperf_filter_value fval = { i, 0 };
 565
 566                 if (filter_type == BPERF_FILTER_PID ||
 567                     filter_type == BPERF_FILTER_TGID)
 568                         key = perf_thread_map__pid(evsel->core.threads, i);
 569                 else if (filter_type == BPERF_FILTER_CPU)
 570                         key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu;
 571                 else
 572                         break;
 573
 574                 filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter);
 575                 bpf_map_update_elem(filter_map_fd, &key, &fval, BPF_ANY);
 576         }
 577
 578         evsel->follower_skel->bss->type = filter_type;
 579         evsel->follower_skel->bss->inherit = target->inherit;
 580
 581         err = bperf_attach_follower_program(evsel->follower_skel, filter_type,
 582                                             target->inherit);
 583
 584 out:
 585         if (err && evsel->bperf_leader_link_fd >= 0)
 586                 close(evsel->bperf_leader_link_fd);
 587         if (err && evsel->bperf_leader_prog_fd >= 0)
 588                 close(evsel->bperf_leader_prog_fd);
 589         if (diff_map_fd >= 0)
 590                 close(diff_map_fd);
 591
 592         flock(attr_map_fd, LOCK_UN);
 593         close(attr_map_fd);
 594
 595         return err;
 596 }
 597
 598 static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)
 599 {
 600         struct bperf_leader_bpf *skel = evsel->leader_skel;
 601
 602         return bpf_map_update_elem(bpf_map__fd(skel->maps.events),
 603                                    &cpu_map_idx, &fd, BPF_ANY);
 604 }
 605
 606 /*
 607  * trigger the leader prog on each cpu, so the accum_reading map could get
 608  * the latest readings.
 609  */
 610 static int bperf_sync_counters(struct evsel *evsel)
 611 {
 612         int num_cpu, i, cpu;
 613
 614         num_cpu = perf_cpu_map__nr(all_cpu_map);
 615         for (i = 0; i < num_cpu; i++) {
 616                 cpu = perf_cpu_map__cpu(all_cpu_map, i).cpu;
 617                 bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu);
 618         }
 619         return 0;
 620 }
 621
 622 static int bperf__enable(struct evsel *evsel)
 623 {
 624         evsel->follower_skel->bss->enabled = 1;
 625         return 0;
 626 }
 627
 628 static int bperf__disable(struct evsel *evsel)
 629 {
 630         evsel->follower_skel->bss->enabled = 0;
 631         return 0;
 632 }
 633
 634 static int bperf__read(struct evsel *evsel)
 635 {
 636         struct bperf_follower_bpf *skel = evsel->follower_skel;
 637         __u32 num_cpu_bpf = cpu__max_cpu().cpu;
 638         struct bpf_perf_event_value values[num_cpu_bpf];
 639         struct perf_counts_values *counts;
 640         int reading_map_fd, err = 0;
 641         __u32 i;
 642         int j;
 643
 644         bperf_sync_counters(evsel);
 645         reading_map_fd = bpf_map__fd(skel->maps.accum_readings);
 646
 647         for (i = 0; i < filter_entry_cnt; i++) {
 648                 struct perf_cpu entry;
 649                 __u32 cpu;
 650
 651                 err = bpf_map_lookup_elem(reading_map_fd, &i, values);
 652                 if (err)
 653                         goto out;
 654                 switch (evsel->follower_skel->bss->type) {
 655                 case BPERF_FILTER_GLOBAL:
 656                         assert(i == 0);
 657
 658                         perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) {
 659                                 counts = perf_counts(evsel->counts, j, 0);
 660                                 counts->val = values[entry.cpu].counter;
 661                                 counts->ena = values[entry.cpu].enabled;
 662                                 counts->run = values[entry.cpu].running;
 663                         }
 664                         break;
 665                 case BPERF_FILTER_CPU:
 666                         cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu;
 667                         assert(cpu >= 0);
 668                         counts = perf_counts(evsel->counts, i, 0);
 669                         counts->val = values[cpu].counter;
 670                         counts->ena = values[cpu].enabled;
 671                         counts->run = values[cpu].running;
 672                         break;
 673                 case BPERF_FILTER_PID:
 674                 case BPERF_FILTER_TGID:
 675                         counts = perf_counts(evsel->counts, 0, i);
 676                         counts->val = 0;
 677                         counts->ena = 0;
 678                         counts->run = 0;
 679
 680                         for (cpu = 0; cpu < num_cpu_bpf; cpu++) {
 681                                 counts->val += values[cpu].counter;
 682                                 counts->ena += values[cpu].enabled;
 683                                 counts->run += values[cpu].running;
 684                         }
 685                         break;
 686                 default:
 687                         break;
 688                 }
 689         }
 690 out:
 691         return err;
 692 }
 693
 694 static int bperf__destroy(struct evsel *evsel)
 695 {
 696         bperf_follower_bpf__destroy(evsel->follower_skel);
 697         close(evsel->bperf_leader_prog_fd);
 698         close(evsel->bperf_leader_link_fd);
 699         return 0;
 700 }
 701
 702 /*
 703  * bperf: share hardware PMCs with BPF
 704  *
 705  * perf uses performance monitoring counters (PMC) to monitor system
 706  * performance. The PMCs are limited hardware resources. For example,
 707  * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu.
 708  *
 709  * Modern data center systems use these PMCs in many different ways:
 710  * system level monitoring, (maybe nested) container level monitoring, per
 711  * process monitoring, profiling (in sample mode), etc. In some cases,
 712  * there are more active perf_events than available hardware PMCs. To allow
 713  * all perf_events to have a chance to run, it is necessary to do expensive
 714  * time multiplexing of events.
 715  *
 716  * On the other hand, many monitoring tools count the common metrics
 717  * (cycles, instructions). It is a waste to have multiple tools create
 718  * multiple perf_events of "cycles" and occupy multiple PMCs.
 719  *
 720  * bperf tries to reduce such wastes by allowing multiple perf_events of
 721  * "cycles" or "instructions" (at different scopes) to share PMUs. Instead
 722  * of having each perf-stat session to read its own perf_events, bperf uses
 723  * BPF programs to read the perf_events and aggregate readings to BPF maps.
 724  * Then, the perf-stat session(s) reads the values from these BPF maps.
 725  *
 726  *                                ||
 727  *       shared progs and maps <- || -> per session progs and maps
 728  *                                ||
 729  *   ---------------              ||
 730  *   | perf_events |              ||
 731  *   ---------------       fexit  ||      -----------------
 732  *          |             --------||----> | follower prog |
 733  *       --------------- /        || ---  -----------------
 734  * cs -> | leader prog |/         ||/        |         |
 735  *   --> ---------------         /||  --------------  ------------------
 736  *  /       |         |         / ||  | filter map |  | accum_readings |
 737  * /  ------------  ------------  ||  --------------  ------------------
 738  * |  | prev map |  | diff map |  ||                        |
 739  * |  ------------  ------------  ||                        |
 740  *  \                             ||                        |
 741  * = \ ==================================================== | ============
 742  *    \                                                    /   user space
 743  *     \                                                  /
 744  *      \                                                /
 745  *    BPF_PROG_TEST_RUN                    BPF_MAP_LOOKUP_ELEM
 746  *        \                                            /
 747  *         \                                          /
 748  *          \------  perf-stat ----------------------/
 749  *
 750  * The figure above shows the architecture of bperf. Note that the figure
 751  * is divided into 3 regions: shared progs and maps (top left), per session
 752  * progs and maps (top right), and user space (bottom).
 753  *
 754  * The leader prog is triggered on each context switch (cs). The leader
 755  * prog reads perf_events and stores the difference (current_reading -
 756  * previous_reading) to the diff map. For the same metric, e.g. "cycles",
 757  * multiple perf-stat sessions share the same leader prog.
 758  *
 759  * Each perf-stat session creates a follower prog as fexit program to the
 760  * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38)
 761  * follower progs to the same leader prog. The follower prog checks current
 762  * task and processor ID to decide whether to add the value from the diff
 763  * map to its accumulated reading map (accum_readings).
 764  *
 765  * Finally, perf-stat user space reads the value from accum_reading map.
 766  *
 767  * Besides context switch, it is also necessary to trigger the leader prog
 768  * before perf-stat reads the value. Otherwise, the accum_reading map may
 769  * not have the latest reading from the perf_events. This is achieved by
 770  * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU.
 771  *
 772  * Comment before the definition of struct perf_event_attr_map_entry
 773  * describes how different sessions of perf-stat share information about
 774  * the leader prog.
 775  */
 776
 777 struct bpf_counter_ops bperf_ops = {
 778         .load       = bperf__load,
 779         .enable     = bperf__enable,
 780         .disable    = bperf__disable,
 781         .read       = bperf__read,
 782         .install_pe = bperf__install_pe,
 783         .destroy    = bperf__destroy,
 784 };
 785
 786 extern struct bpf_counter_ops bperf_cgrp_ops;
 787
 788 static inline bool bpf_counter_skip(struct evsel *evsel)
 789 {
 790         return evsel->bpf_counter_ops == NULL;
 791 }
 792
 793 int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd)
 794 {
 795         if (bpf_counter_skip(evsel))
 796                 return 0;
 797         return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd);
 798 }
 799
 800 int bpf_counter__load(struct evsel *evsel, struct target *target)
 801 {
 802         if (target->bpf_str)
 803                 evsel->bpf_counter_ops = &bpf_program_profiler_ops;
 804         else if (cgrp_event_expanded && target->use_bpf)
 805                 evsel->bpf_counter_ops = &bperf_cgrp_ops;
 806         else if (target->use_bpf || evsel->bpf_counter ||
 807                  evsel__match_bpf_counter_events(evsel->name))
 808                 evsel->bpf_counter_ops = &bperf_ops;
 809
 810         if (evsel->bpf_counter_ops)
 811                 return evsel->bpf_counter_ops->load(evsel, target);
 812         return 0;
 813 }
 814
 815 int bpf_counter__enable(struct evsel *evsel)
 816 {
 817         if (bpf_counter_skip(evsel))
 818                 return 0;
 819         return evsel->bpf_counter_ops->enable(evsel);
 820 }
 821
 822 int bpf_counter__disable(struct evsel *evsel)
 823 {
 824         if (bpf_counter_skip(evsel))
 825                 return 0;
 826         return evsel->bpf_counter_ops->disable(evsel);
 827 }
 828
 829 int bpf_counter__read(struct evsel *evsel)
 830 {
 831         if (bpf_counter_skip(evsel))
 832                 return -EAGAIN;
 833         return evsel->bpf_counter_ops->read(evsel);
 834 }
 835
 836 void bpf_counter__destroy(struct evsel *evsel)
 837 {
 838         if (bpf_counter_skip(evsel))
 839                 return;
 840         evsel->bpf_counter_ops->destroy(evsel);
 841         evsel->bpf_counter_ops = NULL;
 842         evsel->bpf_skel = NULL;
 843 }