tools/perf/tests/shell/test_intel_pt.sh

   1 #!/bin/sh
   2 # Miscellaneous Intel PT testing (exclusive)
   3 # SPDX-License-Identifier: GPL-2.0
   4
   5 set -e
   6
   7 # Skip if no Intel PT
   8 perf list pmu | grep -q 'intel_pt//' || exit 2
   9
  10 shelldir=$(dirname "$0")
  11 # shellcheck source=lib/waiting.sh
  12 . "${shelldir}"/lib/waiting.sh
  13
  14 skip_cnt=0
  15 ok_cnt=0
  16 err_cnt=0
  17
  18 temp_dir=$(mktemp -d /tmp/perf-test-intel-pt-sh.XXXXXXXXXX)
  19
  20 tmpfile="${temp_dir}/tmp-perf.data"
  21 perfdatafile="${temp_dir}/test-perf.data"
  22 outfile="${temp_dir}/test-out.txt"
  23 errfile="${temp_dir}/test-err.txt"
  24 workload="${temp_dir}/workload"
  25 awkscript="${temp_dir}/awkscript"
  26 jitdump_workload="${temp_dir}/jitdump_workload"
  27 maxbrstack="${temp_dir}/maxbrstack.py"
  28
  29 cleanup()
  30 {
  31         trap - EXIT TERM INT
  32         sane=$(echo "${temp_dir}" | cut -b 1-26)
  33         if [ "${sane}" = "/tmp/perf-test-intel-pt-sh" ] ; then
  34                 echo "--- Cleaning up ---"
  35                 rm -f "${temp_dir}/"*
  36                 rmdir "${temp_dir}"
  37         fi
  38 }
  39
  40 trap_cleanup()
  41 {
  42         cleanup
  43         exit 1
  44 }
  45
  46 trap trap_cleanup EXIT TERM INT
  47
  48 # perf record for testing without decoding
  49 perf_record_no_decode()
  50 {
  51         # Options to speed up recording: no post-processing, no build-id cache update,
  52         # and no BPF events.
  53         perf record -B -N --no-bpf-event "$@"
  54 }
  55
  56 # perf record for testing should not need BPF events
  57 perf_record_no_bpf()
  58 {
  59         # Options for no BPF events
  60         perf record --no-bpf-event "$@"
  61 }
  62
  63 have_workload=false
  64 cat << _end_of_file_ | /usr/bin/cc -o "${workload}" -xc - -pthread && have_workload=true
  65 #include <time.h>
  66 #include <pthread.h>
  67
  68 void work(void) {
  69         struct timespec tm = {
  70                 .tv_nsec = 1000000,
  71         };
  72         int i;
  73
  74         /* Run for about 30 seconds */
  75         for (i = 0; i < 30000; i++)
  76                 nanosleep(&tm, NULL);
  77 }
  78
  79 void *threadfunc(void *arg) {
  80         work();
  81         return NULL;
  82 }
  83
  84 int main(void) {
  85         pthread_t th;
  86
  87         pthread_create(&th, NULL, threadfunc, NULL);
  88         work();
  89         pthread_join(th, NULL);
  90         return 0;
  91 }
  92 _end_of_file_
  93
  94 can_cpu_wide()
  95 {
  96         echo "Checking for CPU-wide recording on CPU $1"
  97         if ! perf_record_no_decode -o "${tmpfile}" -e dummy:u -C "$1" true >/dev/null 2>&1 ; then
  98                 echo "No so skipping"
  99                 return 2
 100         fi
 101         echo OK
 102         return 0
 103 }
 104
 105 test_system_wide_side_band()
 106 {
 107         echo "--- Test system-wide sideband ---"
 108
 109         # Need CPU 0 and CPU 1
 110         can_cpu_wide 0 || return $?
 111         can_cpu_wide 1 || return $?
 112
 113         # Record on CPU 0 a task running on CPU 1
 114         perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u -C 0 -- taskset --cpu-list 1 uname
 115
 116         # Should get MMAP events from CPU 1 because they can be needed to decode
 117         mmap_cnt=$(perf script -i "${perfdatafile}" --no-itrace --show-mmap-events -C 1 2>/dev/null | grep -c MMAP)
 118
 119         if [ "${mmap_cnt}" -gt 0 ] ; then
 120                 echo OK
 121                 return 0
 122         fi
 123
 124         echo "Failed to record MMAP events on CPU 1 when tracing CPU 0"
 125         return 1
 126 }
 127
 128 can_kernel()
 129 {
 130         if [ -z "${can_kernel_trace}" ] ; then
 131                 can_kernel_trace=0
 132                 perf_record_no_decode -o "${tmpfile}" -e dummy:k true >/dev/null 2>&1 && can_kernel_trace=1
 133         fi
 134         if [ ${can_kernel_trace} -eq 0 ] ; then
 135                 echo "SKIP: no kernel tracing"
 136                 return 2
 137         fi
 138         return 0
 139 }
 140
 141 test_per_thread()
 142 {
 143         k="$1"
 144         desc="$2"
 145
 146         echo "--- Test per-thread ${desc}recording ---"
 147
 148         if ! $have_workload ; then
 149                 echo "No workload, so skipping"
 150                 return 2
 151         fi
 152
 153         if [ "${k}" = "k" ] ; then
 154                 can_kernel || return 2
 155         fi
 156
 157         cat <<- "_end_of_file_" > "${awkscript}"
 158         BEGIN {
 159                 s = "[ ]*"
 160                 u = s"[0-9]+"s
 161                 d = s"[0-9-]+"s
 162                 x = s"[0-9a-fA-FxX]+"s
 163                 mmapping = "idx"u": mmapping fd"u
 164                 set_output = "idx"u": set output fd"u"->"u
 165                 perf_event_open = "sys_perf_event_open: pid"d"cpu"d"group_fd"d"flags"x"="u
 166         }
 167
 168         /perf record opening and mmapping events/ {
 169                 if (!done)
 170                         active = 1
 171         }
 172
 173         /perf record done opening and mmapping events/ {
 174                 active = 0
 175                 done = 1
 176         }
 177
 178         $0 ~ perf_event_open && active {
 179                 match($0, perf_event_open)
 180                 $0 = substr($0, RSTART, RLENGTH)
 181                 pid = $3
 182                 cpu = $5
 183                 fd = $11
 184                 print "pid " pid " cpu " cpu " fd " fd " : " $0
 185                 fd_array[fd] = fd
 186                 pid_array[fd] = pid
 187                 cpu_array[fd] = cpu
 188         }
 189
 190         $0 ~ mmapping && active  {
 191                 match($0, mmapping)
 192                 $0 = substr($0, RSTART, RLENGTH)
 193                 fd = $5
 194                 print "fd " fd " : " $0
 195                 if (fd in fd_array) {
 196                         mmap_array[fd] = 1
 197                 } else {
 198                         print "Unknown fd " fd
 199                         exit 1
 200                 }
 201         }
 202
 203         $0 ~ set_output && active {
 204                 match($0, set_output)
 205                 $0 = substr($0, RSTART, RLENGTH)
 206                 fd = $6
 207                 fd_to = $8
 208                 print "fd " fd " fd_to " fd_to " : " $0
 209                 if (fd in fd_array) {
 210                         if (fd_to in fd_array) {
 211                                 set_output_array[fd] = fd_to
 212                         } else {
 213                                 print "Unknown fd " fd_to
 214                                 exit 1
 215                         }
 216                 } else {
 217                         print "Unknown fd " fd
 218                         exit 1
 219                 }
 220         }
 221
 222         END {
 223                 print "Checking " length(fd_array) " fds"
 224                 for (fd in fd_array) {
 225                         if (fd in mmap_array) {
 226                                 pid = pid_array[fd]
 227                                 if (pid != -1) {
 228                                         if (pid in pids) {
 229                                                 print "More than 1 mmap for PID " pid
 230                                                 exit 1
 231                                         }
 232                                         pids[pid] = 1
 233                                 }
 234                                 cpu = cpu_array[fd]
 235                                 if (cpu != -1) {
 236                                         if (cpu in cpus) {
 237                                                 print "More than 1 mmap for CPU " cpu
 238                                                 exit 1
 239                                         }
 240                                         cpus[cpu] = 1
 241                                 }
 242                         } else if (!(fd in set_output_array)) {
 243                                 print "No mmap for fd " fd
 244                                 exit 1
 245                         }
 246                 }
 247                 n = length(pids)
 248                 if (n != thread_cnt) {
 249                         print "Expected " thread_cnt " per-thread mmaps - found " n
 250                         exit 1
 251                 }
 252         }
 253         _end_of_file_
 254
 255         $workload &
 256         w1=$!
 257         $workload &
 258         w2=$!
 259         echo "Workload PIDs are $w1 and $w2"
 260         wait_for_threads ${w1} 2
 261         wait_for_threads ${w2} 2
 262
 263         perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u"${k}" -vvv --per-thread -p "${w1},${w2}" 2>"${errfile}" >"${outfile}" &
 264         ppid=$!
 265         echo "perf PID is $ppid"
 266         wait_for_perf_to_start ${ppid} "${errfile}" || return 1
 267
 268         kill ${w1}
 269         wait_for_process_to_exit ${w1} || return 1
 270         is_running ${ppid} || return 1
 271
 272         kill ${w2}
 273         wait_for_process_to_exit ${w2} || return 1
 274         wait_for_process_to_exit ${ppid} || return 1
 275
 276         awk -v thread_cnt=4 -f "${awkscript}" "${errfile}" || return 1
 277
 278         echo OK
 279         return 0
 280 }
 281
 282 test_jitdump()
 283 {
 284         echo "--- Test tracing self-modifying code that uses jitdump ---"
 285
 286         script_path=$(realpath "$0")
 287         script_dir=$(dirname "$script_path")
 288         jitdump_incl_dir="${script_dir}/../../util"
 289         jitdump_h="${jitdump_incl_dir}/jitdump.h"
 290
 291         if [ ! -e "${jitdump_h}" ] ; then
 292                 echo "SKIP: Include file jitdump.h not found"
 293                 return 2
 294         fi
 295
 296         if [ -z "${have_jitdump_workload}" ] ; then
 297                 have_jitdump_workload=false
 298                 # Create a workload that uses self-modifying code and generates its own jitdump file
 299                 cat <<- "_end_of_file_" | /usr/bin/cc -o "${jitdump_workload}" -I "${jitdump_incl_dir}" -xc - -pthread && have_jitdump_workload=true
 300                 #define _GNU_SOURCE
 301                 #include <sys/mman.h>
 302                 #include <sys/types.h>
 303                 #include <stddef.h>
 304                 #include <stdio.h>
 305                 #include <stdint.h>
 306                 #include <unistd.h>
 307                 #include <string.h>
 308
 309                 #include "jitdump.h"
 310
 311                 #define CHK_BYTE 0x5a
 312
 313                 static inline uint64_t rdtsc(void)
 314                 {
 315                         unsigned int low, high;
 316
 317                         asm volatile("rdtsc" : "=a" (low), "=d" (high));
 318
 319                         return low | ((uint64_t)high) << 32;
 320                 }
 321
 322                 static FILE *open_jitdump(void)
 323                 {
 324                         struct jitheader header = {
 325                                 .magic      = JITHEADER_MAGIC,
 326                                 .version    = JITHEADER_VERSION,
 327                                 .total_size = sizeof(header),
 328                                 .pid        = getpid(),
 329                                 .timestamp  = rdtsc(),
 330                                 .flags      = JITDUMP_FLAGS_ARCH_TIMESTAMP,
 331                         };
 332                         char filename[256];
 333                         FILE *f;
 334                         void *m;
 335
 336                         snprintf(filename, sizeof(filename), "jit-%d.dump", getpid());
 337                         f = fopen(filename, "w+");
 338                         if (!f)
 339                                 goto err;
 340                         /* Create an MMAP event for the jitdump file. That is how perf tool finds it. */
 341                         m = mmap(0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(f), 0);
 342                         if (m == MAP_FAILED)
 343                                 goto err_close;
 344                         munmap(m, 4096);
 345                         if (fwrite(&header,sizeof(header),1,f) != 1)
 346                                 goto err_close;
 347                         return f;
 348
 349                 err_close:
 350                         fclose(f);
 351                 err:
 352                         return NULL;
 353                 }
 354
 355                 static int write_jitdump(FILE *f, void *addr, const uint8_t *dat, size_t sz, uint64_t *idx)
 356                 {
 357                         struct jr_code_load rec = {
 358                                 .p.id          = JIT_CODE_LOAD,
 359                                 .p.total_size  = sizeof(rec) + sz,
 360                                 .p.timestamp   = rdtsc(),
 361                                 .pid           = getpid(),
 362                                 .tid           = gettid(),
 363                                 .vma           = (unsigned long)addr,
 364                                 .code_addr     = (unsigned long)addr,
 365                                 .code_size     = sz,
 366                                 .code_index    = ++*idx,
 367                         };
 368
 369                         if (fwrite(&rec,sizeof(rec),1,f) != 1 ||
 370                         fwrite(dat, sz, 1, f) != 1)
 371                                 return -1;
 372                         return 0;
 373                 }
 374
 375                 static void close_jitdump(FILE *f)
 376                 {
 377                         fclose(f);
 378                 }
 379
 380                 int main()
 381                 {
 382                         /* Get a memory page to store executable code */
 383                         void *addr = mmap(0, 4096, PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 384                         /* Code to execute: mov CHK_BYTE, %eax ; ret */
 385                         uint8_t dat[] = {0xb8, CHK_BYTE, 0x00, 0x00, 0x00, 0xc3};
 386                         FILE *f = open_jitdump();
 387                         uint64_t idx = 0;
 388                         int ret = 1;
 389
 390                         if (!f)
 391                                 return 1;
 392                         /* Copy executable code to executable memory page */
 393                         memcpy(addr, dat, sizeof(dat));
 394                         /* Record it in the jitdump file */
 395                         if (write_jitdump(f, addr, dat, sizeof(dat), &idx))
 396                                 goto out_close;
 397                         /* Call it */
 398                         ret = ((int (*)(void))addr)() - CHK_BYTE;
 399                 out_close:
 400                         close_jitdump(f);
 401                         return ret;
 402                 }
 403                 _end_of_file_
 404         fi
 405
 406         if ! $have_jitdump_workload ; then
 407                 echo "SKIP: No jitdump workload"
 408                 return 2
 409         fi
 410
 411         # Change to temp_dir so jitdump collateral files go there
 412         cd "${temp_dir}"
 413         perf_record_no_bpf -o "${tmpfile}" -e intel_pt//u "${jitdump_workload}"
 414         perf inject -i "${tmpfile}" -o "${perfdatafile}" --jit
 415         decode_br_cnt=$(perf script -i "${perfdatafile}" --itrace=b | wc -l)
 416         # Note that overflow and lost errors are suppressed for the error count
 417         decode_err_cnt=$(perf script -i "${perfdatafile}" --itrace=e-o-l | grep -ci error)
 418         cd -
 419         # Should be thousands of branches
 420         if [ "${decode_br_cnt}" -lt 1000 ] ; then
 421                 echo "Decode failed, only ${decode_br_cnt} branches"
 422                 return 1
 423         fi
 424         # Should be no errors
 425         if [ "${decode_err_cnt}" -ne 0 ] ; then
 426                 echo "Decode failed, ${decode_err_cnt} errors"
 427                 perf script -i "${perfdatafile}" --itrace=e-o-l --show-mmap-events | cat
 428                 return 1
 429         fi
 430
 431         echo OK
 432         return 0
 433 }
 434
 435 test_packet_filter()
 436 {
 437         echo "--- Test with MTC and TSC disabled ---"
 438         # Disable MTC and TSC
 439         perf_record_no_decode -o "${perfdatafile}" -e intel_pt/mtc=0,tsc=0/u uname
 440         # Should not get MTC packet
 441         mtc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "MTC 0x")
 442         if [ "${mtc_cnt}" -ne 0 ] ; then
 443                 echo "Failed to filter with mtc=0"
 444                 return 1
 445         fi
 446         # Should not get TSC package
 447         tsc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TSC 0x")
 448         if [ "${tsc_cnt}" -ne 0 ] ; then
 449                 echo "Failed to filter with tsc=0"
 450                 return 1
 451         fi
 452         echo OK
 453         return 0
 454 }
 455
 456 test_disable_branch()
 457 {
 458         echo "--- Test with branches disabled ---"
 459         # Disable branch
 460         perf_record_no_decode -o "${perfdatafile}" -e intel_pt/branch=0/u uname
 461         # Should not get branch related packets
 462         tnt_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TNT 0x")
 463         tip_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TIP 0x")
 464         fup_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "FUP 0x")
 465         if [ "${tnt_cnt}" -ne 0 ] || [ "${tip_cnt}" -ne 0 ] || [ "${fup_cnt}" -ne 0 ] ; then
 466                 echo "Failed to disable branches"
 467                 return 1
 468         fi
 469         echo OK
 470         return 0
 471 }
 472
 473 test_time_cyc()
 474 {
 475         echo "--- Test with/without CYC ---"
 476         # Check if CYC is supported
 477         cyc=$(cat /sys/bus/event_source/devices/intel_pt/caps/psb_cyc)
 478         if [ "${cyc}" != "1" ] ; then
 479                 echo "SKIP: CYC is not supported"
 480                 return 2
 481         fi
 482         # Enable CYC
 483         perf_record_no_decode -o "${perfdatafile}" -e intel_pt/cyc/u uname
 484         # should get CYC packets
 485         cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x")
 486         if [ "${cyc_cnt}" = "0" ] ; then
 487                 echo "Failed to get CYC packet"
 488                 return 1
 489         fi
 490         # Without CYC
 491         perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u uname
 492         # Should not get CYC packets
 493         cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x")
 494         if [ "${cyc_cnt}" -gt 0 ] ; then
 495                 echo "Still get CYC packet without cyc"
 496                 return 1
 497         fi
 498         echo OK
 499         return 0
 500 }
 501
 502 test_sample()
 503 {
 504         echo "--- Test recording with sample mode ---"
 505         # Check if recording with sample mode is working
 506         if ! perf_record_no_decode -o "${perfdatafile}" --aux-sample=8192 -e '{intel_pt//u,branch-misses:u}' uname ; then
 507                 echo "perf record failed with --aux-sample"
 508                 return 1
 509         fi
 510         # Check with event with PMU name
 511         if perf_record_no_decode -o "${perfdatafile}" -e br_misp_retired.all_branches:u uname ; then
 512                 if ! perf_record_no_decode -o "${perfdatafile}" -e '{intel_pt//,br_misp_retired.all_branches/aux-sample-size=8192/}:u' uname ; then
 513                         echo "perf record failed with --aux-sample-size"
 514                         return 1
 515                 fi
 516         fi
 517         echo OK
 518         return 0
 519 }
 520
 521 test_kernel_trace()
 522 {
 523         echo "--- Test with kernel trace ---"
 524         # Check if recording with kernel trace is working
 525         can_kernel || return 2
 526         if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt//k -m1,128 uname ; then
 527                 echo "perf record failed with intel_pt//k"
 528                 return 1
 529         fi
 530         echo OK
 531         return 0
 532 }
 533
 534 test_virtual_lbr()
 535 {
 536         echo "--- Test virtual LBR ---"
 537         # Check if python script is supported
 538         libpython=$(perf version --build-options | grep python | grep -cv OFF)
 539         if [ "${libpython}" != "1" ] ; then
 540                 echo "SKIP: python scripting is not supported"
 541                 return 2
 542         fi
 543
 544         # Python script to determine the maximum size of branch stacks
 545         cat << "_end_of_file_" > "${maxbrstack}"
 546 from __future__ import print_function
 547
 548 bmax = 0
 549
 550 def process_event(param_dict):
 551         if "brstack" in param_dict:
 552                 brstack = param_dict["brstack"]
 553                 n = len(brstack)
 554                 global bmax
 555                 if n > bmax:
 556                         bmax = n
 557
 558 def trace_end():
 559         print("max brstack", bmax)
 560 _end_of_file_
 561
 562         # Check if virtual lbr is working
 563         perf_record_no_bpf -o "${perfdatafile}" --aux-sample -e '{intel_pt//,cycles}:u' uname
 564         times_val=$(perf script -i "${perfdatafile}" --itrace=L -s "${maxbrstack}" 2>/dev/null | grep "max brstack " | cut -d " " -f 3)
 565         case "${times_val}" in
 566                 [0-9]*) ;;
 567                 *)      times_val=0;;
 568         esac
 569         if [ "${times_val}" -lt 2 ] ; then
 570                 echo "Failed with virtual lbr"
 571                 return 1
 572         fi
 573         echo OK
 574         return 0
 575 }
 576
 577 test_power_event()
 578 {
 579         echo "--- Test power events ---"
 580         # Check if power events are supported
 581         power_event=$(cat /sys/bus/event_source/devices/intel_pt/caps/power_event_trace)
 582         if [ "${power_event}" != "1" ] ; then
 583                 echo "SKIP: power_event_trace is not supported"
 584                 return 2
 585         fi
 586         if ! perf_record_no_decode -o "${perfdatafile}" -a -e intel_pt/pwr_evt/u uname ; then
 587                 echo "perf record failed with pwr_evt"
 588                 return 1
 589         fi
 590         echo OK
 591         return 0
 592 }
 593
 594 test_no_tnt()
 595 {
 596         echo "--- Test with TNT packets disabled  ---"
 597         # Check if TNT disable is supported
 598         notnt=$(cat /sys/bus/event_source/devices/intel_pt/caps/tnt_disable)
 599         if [ "${notnt}" != "1" ] ; then
 600                 echo "SKIP: tnt_disable is not supported"
 601                 return 2
 602         fi
 603         perf_record_no_decode -o "${perfdatafile}" -e intel_pt/notnt/u uname
 604         # Should be no TNT packets
 605         tnt_cnt=$(perf script -i "${perfdatafile}" -D | grep -c TNT)
 606         if [ "${tnt_cnt}" -ne 0 ] ; then
 607                 echo "TNT packets still there after notnt"
 608                 return 1
 609         fi
 610         echo OK
 611         return 0
 612 }
 613
 614 test_event_trace()
 615 {
 616         echo "--- Test with event_trace ---"
 617         # Check if event_trace is supported
 618         event_trace=$(cat /sys/bus/event_source/devices/intel_pt/caps/event_trace)
 619         if [ "${event_trace}" != 1 ] ; then
 620                 echo "SKIP: event_trace is not supported"
 621                 return 2
 622         fi
 623         if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt/event/u uname ; then
 624                 echo "perf record failed with event trace"
 625                 return 1
 626         fi
 627         echo OK
 628         return 0
 629 }
 630
 631 test_pipe()
 632 {
 633         echo "--- Test with pipe mode ---"
 634         # Check if it works with pipe
 635         if ! perf_record_no_bpf -o- -e intel_pt//u uname | perf report -q -i- --itrace=i10000 ; then
 636                 echo "perf record + report failed with pipe mode"
 637                 return 1
 638         fi
 639         if ! perf_record_no_bpf -o- -e intel_pt//u uname | perf inject -b > /dev/null ; then
 640                 echo "perf record + inject failed with pipe mode"
 641                 return 1
 642         fi
 643         echo OK
 644         return 0
 645 }
 646
 647 count_result()
 648 {
 649         if [ "$1" -eq 2 ] ; then
 650                 skip_cnt=$((skip_cnt + 1))
 651                 return
 652         fi
 653         if [ "$1" -eq 0 ] ; then
 654                 ok_cnt=$((ok_cnt + 1))
 655                 return
 656         fi
 657         err_cnt=$((err_cnt + 1))
 658 }
 659
 660 ret=0
 661 test_system_wide_side_band              || ret=$? ; count_result $ret ; ret=0
 662 test_per_thread "" ""                   || ret=$? ; count_result $ret ; ret=0
 663 test_per_thread "k" "(incl. kernel) "   || ret=$? ; count_result $ret ; ret=0
 664 test_jitdump                            || ret=$? ; count_result $ret ; ret=0
 665 test_packet_filter                      || ret=$? ; count_result $ret ; ret=0
 666 test_disable_branch                     || ret=$? ; count_result $ret ; ret=0
 667 test_time_cyc                           || ret=$? ; count_result $ret ; ret=0
 668 test_sample                             || ret=$? ; count_result $ret ; ret=0
 669 test_kernel_trace                       || ret=$? ; count_result $ret ; ret=0
 670 test_virtual_lbr                        || ret=$? ; count_result $ret ; ret=0
 671 test_power_event                        || ret=$? ; count_result $ret ; ret=0
 672 test_no_tnt                             || ret=$? ; count_result $ret ; ret=0
 673 test_event_trace                        || ret=$? ; count_result $ret ; ret=0
 674 test_pipe                               || ret=$? ; count_result $ret ; ret=0
 675
 676 cleanup
 677
 678 echo "--- Done ---"
 679
 680 if [ ${err_cnt} -gt 0 ] ; then
 681         exit 1
 682 fi
 683
 684 if [ ${ok_cnt} -gt 0 ] ; then
 685         exit 0
 686 fi
 687
 688 exit 2