tools/perf/util/bpf_off_cpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "util/bpf_counter.h"
   3 #include "util/debug.h"
   4 #include "util/evsel.h"
   5 #include "util/evlist.h"
   6 #include "util/off_cpu.h"
   7 #include "util/perf-hooks.h"
   8 #include "util/record.h"
   9 #include "util/session.h"
  10 #include "util/target.h"
  11 #include "util/cpumap.h"
  12 #include "util/thread_map.h"
  13 #include "util/cgroup.h"
  14 #include "util/strlist.h"
  15 #include <bpf/bpf.h>
  16
  17 #include "bpf_skel/off_cpu.skel.h"
  18
  19 #define MAX_STACKS  32
  20 #define MAX_PROC  4096
  21 /* we don't need actual timestamp, just want to put the samples at last */
  22 #define OFF_CPU_TIMESTAMP  (~0ull << 32)
  23
  24 static struct off_cpu_bpf *skel;
  25
  26 struct off_cpu_key {
  27         u32 pid;
  28         u32 tgid;
  29         u32 stack_id;
  30         u32 state;
  31         u64 cgroup_id;
  32 };
  33
  34 union off_cpu_data {
  35         struct perf_event_header hdr;
  36         u64 array[1024 / sizeof(u64)];
  37 };
  38
  39 static int off_cpu_config(struct evlist *evlist)
  40 {
  41         struct evsel *evsel;
  42         struct perf_event_attr attr = {
  43                 .type   = PERF_TYPE_SOFTWARE,
  44                 .config = PERF_COUNT_SW_BPF_OUTPUT,
  45                 .size   = sizeof(attr), /* to capture ABI version */
  46         };
  47         char *evname = strdup(OFFCPU_EVENT);
  48
  49         if (evname == NULL)
  50                 return -ENOMEM;
  51
  52         evsel = evsel__new(&attr);
  53         if (!evsel) {
  54                 free(evname);
  55                 return -ENOMEM;
  56         }
  57
  58         evsel->core.attr.freq = 1;
  59         evsel->core.attr.sample_period = 1;
  60         /* off-cpu analysis depends on stack trace */
  61         evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN;
  62
  63         evlist__add(evlist, evsel);
  64
  65         free(evsel->name);
  66         evsel->name = evname;
  67
  68         return 0;
  69 }
  70
  71 static void off_cpu_start(void *arg)
  72 {
  73         struct evlist *evlist = arg;
  74
  75         /* update task filter for the given workload */
  76         if (skel->rodata->has_task && skel->rodata->uses_tgid &&
  77             perf_thread_map__pid(evlist->core.threads, 0) != -1) {
  78                 int fd;
  79                 u32 pid;
  80                 u8 val = 1;
  81
  82                 fd = bpf_map__fd(skel->maps.task_filter);
  83                 pid = perf_thread_map__pid(evlist->core.threads, 0);
  84                 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
  85         }
  86
  87         skel->bss->enabled = 1;
  88 }
  89
  90 static void off_cpu_finish(void *arg __maybe_unused)
  91 {
  92         skel->bss->enabled = 0;
  93         off_cpu_bpf__destroy(skel);
  94 }
  95
  96 /* v5.18 kernel added prev_state arg, so it needs to check the signature */
  97 static void check_sched_switch_args(void)
  98 {
  99         struct btf *btf = btf__load_vmlinux_btf();
 100         const struct btf_type *t1, *t2, *t3;
 101         u32 type_id;
 102
 103         if (!btf) {
 104                 pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n");
 105                 goto cleanup;
 106         }
 107
 108         type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch",
 109                                          BTF_KIND_TYPEDEF);
 110         if ((s32)type_id < 0)
 111                 goto cleanup;
 112
 113         t1 = btf__type_by_id(btf, type_id);
 114         if (t1 == NULL)
 115                 goto cleanup;
 116
 117         t2 = btf__type_by_id(btf, t1->type);
 118         if (t2 == NULL || !btf_is_ptr(t2))
 119                 goto cleanup;
 120
 121         t3 = btf__type_by_id(btf, t2->type);
 122         /* btf_trace func proto has one more argument for the context */
 123         if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) {
 124                 /* new format: pass prev_state as 4th arg */
 125                 skel->rodata->has_prev_state = true;
 126         }
 127 cleanup:
 128         btf__free(btf);
 129 }
 130
 131 int off_cpu_prepare(struct evlist *evlist, struct target *target,
 132                     struct record_opts *opts)
 133 {
 134         int err, fd, i;
 135         int ncpus = 1, ntasks = 1, ncgrps = 1;
 136         struct strlist *pid_slist = NULL;
 137         struct str_node *pos;
 138
 139         if (off_cpu_config(evlist) < 0) {
 140                 pr_err("Failed to config off-cpu BPF event\n");
 141                 return -1;
 142         }
 143
 144         skel = off_cpu_bpf__open();
 145         if (!skel) {
 146                 pr_err("Failed to open off-cpu BPF skeleton\n");
 147                 return -1;
 148         }
 149
 150         /* don't need to set cpu filter for system-wide mode */
 151         if (target->cpu_list) {
 152                 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
 153                 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
 154                 skel->rodata->has_cpu = 1;
 155         }
 156
 157         if (target->pid) {
 158                 pid_slist = strlist__new(target->pid, NULL);
 159                 if (!pid_slist) {
 160                         pr_err("Failed to create a strlist for pid\n");
 161                         return -1;
 162                 }
 163
 164                 ntasks = 0;
 165                 strlist__for_each_entry(pos, pid_slist) {
 166                         char *end_ptr;
 167                         int pid = strtol(pos->s, &end_ptr, 10);
 168
 169                         if (pid == INT_MIN || pid == INT_MAX ||
 170                             (*end_ptr != '\0' && *end_ptr != ','))
 171                                 continue;
 172
 173                         ntasks++;
 174                 }
 175
 176                 if (ntasks < MAX_PROC)
 177                         ntasks = MAX_PROC;
 178
 179                 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 180                 skel->rodata->has_task = 1;
 181                 skel->rodata->uses_tgid = 1;
 182         } else if (target__has_task(target)) {
 183                 ntasks = perf_thread_map__nr(evlist->core.threads);
 184                 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 185                 skel->rodata->has_task = 1;
 186         } else if (target__none(target)) {
 187                 bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC);
 188                 skel->rodata->has_task = 1;
 189                 skel->rodata->uses_tgid = 1;
 190         }
 191
 192         if (evlist__first(evlist)->cgrp) {
 193                 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
 194                 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
 195
 196                 if (!cgroup_is_v2("perf_event"))
 197                         skel->rodata->uses_cgroup_v1 = true;
 198                 skel->rodata->has_cgroup = 1;
 199         }
 200
 201         if (opts->record_cgroup) {
 202                 skel->rodata->needs_cgroup = true;
 203
 204                 if (!cgroup_is_v2("perf_event"))
 205                         skel->rodata->uses_cgroup_v1 = true;
 206         }
 207
 208         set_max_rlimit();
 209         check_sched_switch_args();
 210
 211         err = off_cpu_bpf__load(skel);
 212         if (err) {
 213                 pr_err("Failed to load off-cpu skeleton\n");
 214                 goto out;
 215         }
 216
 217         if (target->cpu_list) {
 218                 u32 cpu;
 219                 u8 val = 1;
 220
 221                 fd = bpf_map__fd(skel->maps.cpu_filter);
 222
 223                 for (i = 0; i < ncpus; i++) {
 224                         cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
 225                         bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 226                 }
 227         }
 228
 229         if (target->pid) {
 230                 u8 val = 1;
 231
 232                 fd = bpf_map__fd(skel->maps.task_filter);
 233
 234                 strlist__for_each_entry(pos, pid_slist) {
 235                         char *end_ptr;
 236                         u32 tgid;
 237                         int pid = strtol(pos->s, &end_ptr, 10);
 238
 239                         if (pid == INT_MIN || pid == INT_MAX ||
 240                             (*end_ptr != '\0' && *end_ptr != ','))
 241                                 continue;
 242
 243                         tgid = pid;
 244                         bpf_map_update_elem(fd, &tgid, &val, BPF_ANY);
 245                 }
 246         } else if (target__has_task(target)) {
 247                 u32 pid;
 248                 u8 val = 1;
 249
 250                 fd = bpf_map__fd(skel->maps.task_filter);
 251
 252                 for (i = 0; i < ntasks; i++) {
 253                         pid = perf_thread_map__pid(evlist->core.threads, i);
 254                         bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 255                 }
 256         }
 257
 258         if (evlist__first(evlist)->cgrp) {
 259                 struct evsel *evsel;
 260                 u8 val = 1;
 261
 262                 fd = bpf_map__fd(skel->maps.cgroup_filter);
 263
 264                 evlist__for_each_entry(evlist, evsel) {
 265                         struct cgroup *cgrp = evsel->cgrp;
 266
 267                         if (cgrp == NULL)
 268                                 continue;
 269
 270                         if (!cgrp->id && read_cgroup_id(cgrp) < 0) {
 271                                 pr_err("Failed to read cgroup id of %s\n",
 272                                        cgrp->name);
 273                                 goto out;
 274                         }
 275
 276                         bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY);
 277                 }
 278         }
 279
 280         err = off_cpu_bpf__attach(skel);
 281         if (err) {
 282                 pr_err("Failed to attach off-cpu BPF skeleton\n");
 283                 goto out;
 284         }
 285
 286         if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) ||
 287             perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) {
 288                 pr_err("Failed to attach off-cpu skeleton\n");
 289                 goto out;
 290         }
 291
 292         return 0;
 293
 294 out:
 295         off_cpu_bpf__destroy(skel);
 296         return -1;
 297 }
 298
 299 int off_cpu_write(struct perf_session *session)
 300 {
 301         int bytes = 0, size;
 302         int fd, stack;
 303         u64 sample_type, val, sid = 0;
 304         struct evsel *evsel;
 305         struct perf_data_file *file = &session->data->file;
 306         struct off_cpu_key prev, key;
 307         union off_cpu_data data = {
 308                 .hdr = {
 309                         .type = PERF_RECORD_SAMPLE,
 310                         .misc = PERF_RECORD_MISC_USER,
 311                 },
 312         };
 313         u64 tstamp = OFF_CPU_TIMESTAMP;
 314
 315         skel->bss->enabled = 0;
 316
 317         evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT);
 318         if (evsel == NULL) {
 319                 pr_err("%s evsel not found\n", OFFCPU_EVENT);
 320                 return 0;
 321         }
 322
 323         sample_type = evsel->core.attr.sample_type;
 324
 325         if (sample_type & ~OFFCPU_SAMPLE_TYPES) {
 326                 pr_err("not supported sample type: %llx\n",
 327                        (unsigned long long)sample_type);
 328                 return -1;
 329         }
 330
 331         if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) {
 332                 if (evsel->core.id)
 333                         sid = evsel->core.id[0];
 334         }
 335
 336         fd = bpf_map__fd(skel->maps.off_cpu);
 337         stack = bpf_map__fd(skel->maps.stacks);
 338         memset(&prev, 0, sizeof(prev));
 339
 340         while (!bpf_map_get_next_key(fd, &prev, &key)) {
 341                 int n = 1;  /* start from perf_event_header */
 342                 int ip_pos = -1;
 343
 344                 bpf_map_lookup_elem(fd, &key, &val);
 345
 346                 if (sample_type & PERF_SAMPLE_IDENTIFIER)
 347                         data.array[n++] = sid;
 348                 if (sample_type & PERF_SAMPLE_IP) {
 349                         ip_pos = n;
 350                         data.array[n++] = 0;  /* will be updated */
 351                 }
 352                 if (sample_type & PERF_SAMPLE_TID)
 353                         data.array[n++] = (u64)key.pid << 32 | key.tgid;
 354                 if (sample_type & PERF_SAMPLE_TIME)
 355                         data.array[n++] = tstamp;
 356                 if (sample_type & PERF_SAMPLE_ID)
 357                         data.array[n++] = sid;
 358                 if (sample_type & PERF_SAMPLE_CPU)
 359                         data.array[n++] = 0;
 360                 if (sample_type & PERF_SAMPLE_PERIOD)
 361                         data.array[n++] = val;
 362                 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 363                         int len = 0;
 364
 365                         /* data.array[n] is callchain->nr (updated later) */
 366                         data.array[n + 1] = PERF_CONTEXT_USER;
 367                         data.array[n + 2] = 0;
 368
 369                         bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]);
 370                         while (data.array[n + 2 + len])
 371                                 len++;
 372
 373                         /* update length of callchain */
 374                         data.array[n] = len + 1;
 375
 376                         /* update sample ip with the first callchain entry */
 377                         if (ip_pos >= 0)
 378                                 data.array[ip_pos] = data.array[n + 2];
 379
 380                         /* calculate sample callchain data array length */
 381                         n += len + 2;
 382                 }
 383                 if (sample_type & PERF_SAMPLE_CGROUP)
 384                         data.array[n++] = key.cgroup_id;
 385
 386                 size = n * sizeof(u64);
 387                 data.hdr.size = size;
 388                 bytes += size;
 389
 390                 if (perf_data_file__write(file, &data, size) < 0) {
 391                         pr_err("failed to write perf data, error: %m\n");
 392                         return bytes;
 393                 }
 394
 395                 prev = key;
 396                 /* increase dummy timestamp to sort later samples */
 397                 tstamp++;
 398         }
 399         return bytes;
 400 }