tools/lib/perf/evlist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <perf/evlist.h>
   3 #include <perf/evsel.h>
   4 #include <linux/bitops.h>
   5 #include <linux/list.h>
   6 #include <linux/hash.h>
   7 #include <sys/ioctl.h>
   8 #include <internal/evlist.h>
   9 #include <internal/evsel.h>
  10 #include <internal/xyarray.h>
  11 #include <internal/mmap.h>
  12 #include <internal/cpumap.h>
  13 #include <internal/threadmap.h>
  14 #include <internal/lib.h>
  15 #include <linux/zalloc.h>
  16 #include <stdlib.h>
  17 #include <errno.h>
  18 #include <unistd.h>
  19 #include <fcntl.h>
  20 #include <signal.h>
  21 #include <poll.h>
  22 #include <sys/mman.h>
  23 #include <perf/cpumap.h>
  24 #include <perf/threadmap.h>
  25 #include <api/fd/array.h>
  26
  27 void perf_evlist__init(struct perf_evlist *evlist)
  28 {
  29         int i;
  30
  31         for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i)
  32                 INIT_HLIST_HEAD(&evlist->heads[i]);
  33         INIT_LIST_HEAD(&evlist->entries);
  34         evlist->nr_entries = 0;
  35         fdarray__init(&evlist->pollfd, 64);
  36 }
  37
  38 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
  39                                           struct perf_evsel *evsel)
  40 {
  41         /*
  42          * We already have cpus for evsel (via PMU sysfs) so
  43          * keep it, if there's no target cpu list defined.
  44          */
  45         if (!evsel->own_cpus || evlist->has_user_cpus) {
  46                 perf_cpu_map__put(evsel->cpus);
  47                 evsel->cpus = perf_cpu_map__get(evlist->cpus);
  48         } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->cpus)) {
  49                 perf_cpu_map__put(evsel->cpus);
  50                 evsel->cpus = perf_cpu_map__get(evlist->cpus);
  51         } else if (evsel->cpus != evsel->own_cpus) {
  52                 perf_cpu_map__put(evsel->cpus);
  53                 evsel->cpus = perf_cpu_map__get(evsel->own_cpus);
  54         }
  55
  56         perf_thread_map__put(evsel->threads);
  57         evsel->threads = perf_thread_map__get(evlist->threads);
  58         evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus);
  59 }
  60
  61 static void perf_evlist__propagate_maps(struct perf_evlist *evlist)
  62 {
  63         struct perf_evsel *evsel;
  64
  65         perf_evlist__for_each_evsel(evlist, evsel)
  66                 __perf_evlist__propagate_maps(evlist, evsel);
  67 }
  68
  69 void perf_evlist__add(struct perf_evlist *evlist,
  70                       struct perf_evsel *evsel)
  71 {
  72         list_add_tail(&evsel->node, &evlist->entries);
  73         evlist->nr_entries += 1;
  74         __perf_evlist__propagate_maps(evlist, evsel);
  75 }
  76
  77 void perf_evlist__remove(struct perf_evlist *evlist,
  78                          struct perf_evsel *evsel)
  79 {
  80         list_del_init(&evsel->node);
  81         evlist->nr_entries -= 1;
  82 }
  83
  84 struct perf_evlist *perf_evlist__new(void)
  85 {
  86         struct perf_evlist *evlist = zalloc(sizeof(*evlist));
  87
  88         if (evlist != NULL)
  89                 perf_evlist__init(evlist);
  90
  91         return evlist;
  92 }
  93
  94 struct perf_evsel *
  95 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev)
  96 {
  97         struct perf_evsel *next;
  98
  99         if (!prev) {
 100                 next = list_first_entry(&evlist->entries,
 101                                         struct perf_evsel,
 102                                         node);
 103         } else {
 104                 next = list_next_entry(prev, node);
 105         }
 106
 107         /* Empty list is noticed here so don't need checking on entry. */
 108         if (&next->node == &evlist->entries)
 109                 return NULL;
 110
 111         return next;
 112 }
 113
 114 static void perf_evlist__purge(struct perf_evlist *evlist)
 115 {
 116         struct perf_evsel *pos, *n;
 117
 118         perf_evlist__for_each_entry_safe(evlist, n, pos) {
 119                 list_del_init(&pos->node);
 120                 perf_evsel__delete(pos);
 121         }
 122
 123         evlist->nr_entries = 0;
 124 }
 125
 126 void perf_evlist__exit(struct perf_evlist *evlist)
 127 {
 128         perf_cpu_map__put(evlist->cpus);
 129         perf_cpu_map__put(evlist->all_cpus);
 130         perf_thread_map__put(evlist->threads);
 131         evlist->cpus = NULL;
 132         evlist->all_cpus = NULL;
 133         evlist->threads = NULL;
 134         fdarray__exit(&evlist->pollfd);
 135 }
 136
 137 void perf_evlist__delete(struct perf_evlist *evlist)
 138 {
 139         if (evlist == NULL)
 140                 return;
 141
 142         perf_evlist__munmap(evlist);
 143         perf_evlist__close(evlist);
 144         perf_evlist__purge(evlist);
 145         perf_evlist__exit(evlist);
 146         free(evlist);
 147 }
 148
 149 void perf_evlist__set_maps(struct perf_evlist *evlist,
 150                            struct perf_cpu_map *cpus,
 151                            struct perf_thread_map *threads)
 152 {
 153         /*
 154          * Allow for the possibility that one or another of the maps isn't being
 155          * changed i.e. don't put it.  Note we are assuming the maps that are
 156          * being applied are brand new and evlist is taking ownership of the
 157          * original reference count of 1.  If that is not the case it is up to
 158          * the caller to increase the reference count.
 159          */
 160         if (cpus != evlist->cpus) {
 161                 perf_cpu_map__put(evlist->cpus);
 162                 evlist->cpus = perf_cpu_map__get(cpus);
 163         }
 164
 165         if (threads != evlist->threads) {
 166                 perf_thread_map__put(evlist->threads);
 167                 evlist->threads = perf_thread_map__get(threads);
 168         }
 169
 170         if (!evlist->all_cpus && cpus)
 171                 evlist->all_cpus = perf_cpu_map__get(cpus);
 172
 173         perf_evlist__propagate_maps(evlist);
 174 }
 175
 176 int perf_evlist__open(struct perf_evlist *evlist)
 177 {
 178         struct perf_evsel *evsel;
 179         int err;
 180
 181         perf_evlist__for_each_entry(evlist, evsel) {
 182                 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads);
 183                 if (err < 0)
 184                         goto out_err;
 185         }
 186
 187         return 0;
 188
 189 out_err:
 190         perf_evlist__close(evlist);
 191         return err;
 192 }
 193
 194 void perf_evlist__close(struct perf_evlist *evlist)
 195 {
 196         struct perf_evsel *evsel;
 197
 198         perf_evlist__for_each_entry_reverse(evlist, evsel)
 199                 perf_evsel__close(evsel);
 200 }
 201
 202 void perf_evlist__enable(struct perf_evlist *evlist)
 203 {
 204         struct perf_evsel *evsel;
 205
 206         perf_evlist__for_each_entry(evlist, evsel)
 207                 perf_evsel__enable(evsel);
 208 }
 209
 210 void perf_evlist__disable(struct perf_evlist *evlist)
 211 {
 212         struct perf_evsel *evsel;
 213
 214         perf_evlist__for_each_entry(evlist, evsel)
 215                 perf_evsel__disable(evsel);
 216 }
 217
 218 u64 perf_evlist__read_format(struct perf_evlist *evlist)
 219 {
 220         struct perf_evsel *first = perf_evlist__first(evlist);
 221
 222         return first->attr.read_format;
 223 }
 224
 225 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
 226
 227 static void perf_evlist__id_hash(struct perf_evlist *evlist,
 228                                  struct perf_evsel *evsel,
 229                                  int cpu, int thread, u64 id)
 230 {
 231         int hash;
 232         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 233
 234         sid->id = id;
 235         sid->evsel = evsel;
 236         hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);
 237         hlist_add_head(&sid->node, &evlist->heads[hash]);
 238 }
 239
 240 void perf_evlist__id_add(struct perf_evlist *evlist,
 241                          struct perf_evsel *evsel,
 242                          int cpu, int thread, u64 id)
 243 {
 244         perf_evlist__id_hash(evlist, evsel, cpu, thread, id);
 245         evsel->id[evsel->ids++] = id;
 246 }
 247
 248 int perf_evlist__id_add_fd(struct perf_evlist *evlist,
 249                            struct perf_evsel *evsel,
 250                            int cpu, int thread, int fd)
 251 {
 252         u64 read_data[4] = { 0, };
 253         int id_idx = 1; /* The first entry is the counter value */
 254         u64 id;
 255         int ret;
 256
 257         ret = ioctl(fd, PERF_EVENT_IOC_ID, &id);
 258         if (!ret)
 259                 goto add;
 260
 261         if (errno != ENOTTY)
 262                 return -1;
 263
 264         /* Legacy way to get event id.. All hail to old kernels! */
 265
 266         /*
 267          * This way does not work with group format read, so bail
 268          * out in that case.
 269          */
 270         if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP)
 271                 return -1;
 272
 273         if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||
 274             read(fd, &read_data, sizeof(read_data)) == -1)
 275                 return -1;
 276
 277         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 278                 ++id_idx;
 279         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 280                 ++id_idx;
 281
 282         id = read_data[id_idx];
 283
 284 add:
 285         perf_evlist__id_add(evlist, evsel, cpu, thread, id);
 286         return 0;
 287 }
 288
 289 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist)
 290 {
 291         int nr_cpus = perf_cpu_map__nr(evlist->cpus);
 292         int nr_threads = perf_thread_map__nr(evlist->threads);
 293         int nfds = 0;
 294         struct perf_evsel *evsel;
 295
 296         perf_evlist__for_each_entry(evlist, evsel) {
 297                 if (evsel->system_wide)
 298                         nfds += nr_cpus;
 299                 else
 300                         nfds += nr_cpus * nr_threads;
 301         }
 302
 303         if (fdarray__available_entries(&evlist->pollfd) < nfds &&
 304             fdarray__grow(&evlist->pollfd, nfds) < 0)
 305                 return -ENOMEM;
 306
 307         return 0;
 308 }
 309
 310 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd,
 311                             void *ptr, short revent, enum fdarray_flags flags)
 312 {
 313         int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags);
 314
 315         if (pos >= 0) {
 316                 evlist->pollfd.priv[pos].ptr = ptr;
 317                 fcntl(fd, F_SETFL, O_NONBLOCK);
 318         }
 319
 320         return pos;
 321 }
 322
 323 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd,
 324                                          void *arg __maybe_unused)
 325 {
 326         struct perf_mmap *map = fda->priv[fd].ptr;
 327
 328         if (map)
 329                 perf_mmap__put(map);
 330 }
 331
 332 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask)
 333 {
 334         return fdarray__filter(&evlist->pollfd, revents_and_mask,
 335                                perf_evlist__munmap_filtered, NULL);
 336 }
 337
 338 int perf_evlist__poll(struct perf_evlist *evlist, int timeout)
 339 {
 340         return fdarray__poll(&evlist->pollfd, timeout);
 341 }
 342
 343 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite)
 344 {
 345         int i;
 346         struct perf_mmap *map;
 347
 348         map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));
 349         if (!map)
 350                 return NULL;
 351
 352         for (i = 0; i < evlist->nr_mmaps; i++) {
 353                 struct perf_mmap *prev = i ? &map[i - 1] : NULL;
 354
 355                 /*
 356                  * When the perf_mmap() call is made we grab one refcount, plus
 357                  * one extra to let perf_mmap__consume() get the last
 358                  * events after all real references (perf_mmap__get()) are
 359                  * dropped.
 360                  *
 361                  * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and
 362                  * thus does perf_mmap__get() on it.
 363                  */
 364                 perf_mmap__init(&map[i], prev, overwrite, NULL);
 365         }
 366
 367         return map;
 368 }
 369
 370 static void perf_evlist__set_sid_idx(struct perf_evlist *evlist,
 371                                      struct perf_evsel *evsel, int idx, int cpu,
 372                                      int thread)
 373 {
 374         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 375
 376         sid->idx = idx;
 377         if (evlist->cpus && cpu >= 0)
 378                 sid->cpu = evlist->cpus->map[cpu];
 379         else
 380                 sid->cpu = -1;
 381         if (!evsel->system_wide && evlist->threads && thread >= 0)
 382                 sid->tid = perf_thread_map__pid(evlist->threads, thread);
 383         else
 384                 sid->tid = -1;
 385 }
 386
 387 static struct perf_mmap*
 388 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx)
 389 {
 390         struct perf_mmap *maps;
 391
 392         maps = overwrite ? evlist->mmap_ovw : evlist->mmap;
 393
 394         if (!maps) {
 395                 maps = perf_evlist__alloc_mmap(evlist, overwrite);
 396                 if (!maps)
 397                         return NULL;
 398
 399                 if (overwrite)
 400                         evlist->mmap_ovw = maps;
 401                 else
 402                         evlist->mmap = maps;
 403         }
 404
 405         return &maps[idx];
 406 }
 407
 408 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y))
 409
 410 static int
 411 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp,
 412                           int output, int cpu)
 413 {
 414         return perf_mmap__mmap(map, mp, output, cpu);
 415 }
 416
 417 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map,
 418                                         bool overwrite)
 419 {
 420         if (overwrite)
 421                 evlist->mmap_ovw_first = map;
 422         else
 423                 evlist->mmap_first = map;
 424 }
 425
 426 static int
 427 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
 428                int idx, struct perf_mmap_param *mp, int cpu_idx,
 429                int thread, int *_output, int *_output_overwrite)
 430 {
 431         int evlist_cpu = perf_cpu_map__cpu(evlist->cpus, cpu_idx);
 432         struct perf_evsel *evsel;
 433         int revent;
 434
 435         perf_evlist__for_each_entry(evlist, evsel) {
 436                 bool overwrite = evsel->attr.write_backward;
 437                 struct perf_mmap *map;
 438                 int *output, fd, cpu;
 439
 440                 if (evsel->system_wide && thread)
 441                         continue;
 442
 443                 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu);
 444                 if (cpu == -1)
 445                         continue;
 446
 447                 map = ops->get(evlist, overwrite, idx);
 448                 if (map == NULL)
 449                         return -ENOMEM;
 450
 451                 if (overwrite) {
 452                         mp->prot = PROT_READ;
 453                         output   = _output_overwrite;
 454                 } else {
 455                         mp->prot = PROT_READ | PROT_WRITE;
 456                         output   = _output;
 457                 }
 458
 459                 fd = FD(evsel, cpu, thread);
 460
 461                 if (*output == -1) {
 462                         *output = fd;
 463
 464                         /*
 465                          * The last one will be done at perf_mmap__consume(), so that we
 466                          * make sure we don't prevent tools from consuming every last event in
 467                          * the ring buffer.
 468                          *
 469                          * I.e. we can get the POLLHUP meaning that the fd doesn't exist
 470                          * anymore, but the last events for it are still in the ring buffer,
 471                          * waiting to be consumed.
 472                          *
 473                          * Tools can chose to ignore this at their own discretion, but the
 474                          * evlist layer can't just drop it when filtering events in
 475                          * perf_evlist__filter_pollfd().
 476                          */
 477                         refcount_set(&map->refcnt, 2);
 478
 479                         if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
 480                                 return -1;
 481
 482                         if (!idx)
 483                                 perf_evlist__set_mmap_first(evlist, map, overwrite);
 484                 } else {
 485                         if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
 486                                 return -1;
 487
 488                         perf_mmap__get(map);
 489                 }
 490
 491                 revent = !overwrite ? POLLIN : 0;
 492
 493                 if (!evsel->system_wide &&
 494                     perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) {
 495                         perf_mmap__put(map);
 496                         return -1;
 497                 }
 498
 499                 if (evsel->attr.read_format & PERF_FORMAT_ID) {
 500                         if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,
 501                                                    fd) < 0)
 502                                 return -1;
 503                         perf_evlist__set_sid_idx(evlist, evsel, idx, cpu,
 504                                                  thread);
 505                 }
 506         }
 507
 508         return 0;
 509 }
 510
 511 static int
 512 mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
 513                 struct perf_mmap_param *mp)
 514 {
 515         int thread;
 516         int nr_threads = perf_thread_map__nr(evlist->threads);
 517
 518         for (thread = 0; thread < nr_threads; thread++) {
 519                 int output = -1;
 520                 int output_overwrite = -1;
 521
 522                 if (ops->idx)
 523                         ops->idx(evlist, mp, thread, false);
 524
 525                 if (mmap_per_evsel(evlist, ops, thread, mp, 0, thread,
 526                                    &output, &output_overwrite))
 527                         goto out_unmap;
 528         }
 529
 530         return 0;
 531
 532 out_unmap:
 533         perf_evlist__munmap(evlist);
 534         return -1;
 535 }
 536
 537 static int
 538 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
 539              struct perf_mmap_param *mp)
 540 {
 541         int nr_threads = perf_thread_map__nr(evlist->threads);
 542         int nr_cpus    = perf_cpu_map__nr(evlist->cpus);
 543         int cpu, thread;
 544
 545         for (cpu = 0; cpu < nr_cpus; cpu++) {
 546                 int output = -1;
 547                 int output_overwrite = -1;
 548
 549                 if (ops->idx)
 550                         ops->idx(evlist, mp, cpu, true);
 551
 552                 for (thread = 0; thread < nr_threads; thread++) {
 553                         if (mmap_per_evsel(evlist, ops, cpu, mp, cpu,
 554                                            thread, &output, &output_overwrite))
 555                                 goto out_unmap;
 556                 }
 557         }
 558
 559         return 0;
 560
 561 out_unmap:
 562         perf_evlist__munmap(evlist);
 563         return -1;
 564 }
 565
 566 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist)
 567 {
 568         int nr_mmaps;
 569
 570         nr_mmaps = perf_cpu_map__nr(evlist->cpus);
 571         if (perf_cpu_map__empty(evlist->cpus))
 572                 nr_mmaps = perf_thread_map__nr(evlist->threads);
 573
 574         return nr_mmaps;
 575 }
 576
 577 int perf_evlist__mmap_ops(struct perf_evlist *evlist,
 578                           struct perf_evlist_mmap_ops *ops,
 579                           struct perf_mmap_param *mp)
 580 {
 581         struct perf_evsel *evsel;
 582         const struct perf_cpu_map *cpus = evlist->cpus;
 583         const struct perf_thread_map *threads = evlist->threads;
 584
 585         if (!ops || !ops->get || !ops->mmap)
 586                 return -EINVAL;
 587
 588         mp->mask = evlist->mmap_len - page_size - 1;
 589
 590         evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist);
 591
 592         perf_evlist__for_each_entry(evlist, evsel) {
 593                 if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
 594                     evsel->sample_id == NULL &&
 595                     perf_evsel__alloc_id(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
 596                         return -ENOMEM;
 597         }
 598
 599         if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
 600                 return -ENOMEM;
 601
 602         if (perf_cpu_map__empty(cpus))
 603                 return mmap_per_thread(evlist, ops, mp);
 604
 605         return mmap_per_cpu(evlist, ops, mp);
 606 }
 607
 608 int perf_evlist__mmap(struct perf_evlist *evlist, int pages)
 609 {
 610         struct perf_mmap_param mp;
 611         struct perf_evlist_mmap_ops ops = {
 612                 .get  = perf_evlist__mmap_cb_get,
 613                 .mmap = perf_evlist__mmap_cb_mmap,
 614         };
 615
 616         evlist->mmap_len = (pages + 1) * page_size;
 617
 618         return perf_evlist__mmap_ops(evlist, &ops, &mp);
 619 }
 620
 621 void perf_evlist__munmap(struct perf_evlist *evlist)
 622 {
 623         int i;
 624
 625         if (evlist->mmap) {
 626                 for (i = 0; i < evlist->nr_mmaps; i++)
 627                         perf_mmap__munmap(&evlist->mmap[i]);
 628         }
 629
 630         if (evlist->mmap_ovw) {
 631                 for (i = 0; i < evlist->nr_mmaps; i++)
 632                         perf_mmap__munmap(&evlist->mmap_ovw[i]);
 633         }
 634
 635         zfree(&evlist->mmap);
 636         zfree(&evlist->mmap_ovw);
 637 }
 638
 639 struct perf_mmap*
 640 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map,
 641                        bool overwrite)
 642 {
 643         if (map)
 644                 return map->next;
 645
 646         return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first;
 647 }