tools/perf/tests/shell/lib/attr.py

   1 # SPDX-License-Identifier: GPL-2.0
   2
   3 from __future__ import print_function
   4
   5 import os
   6 import sys
   7 import glob
   8 import optparse
   9 import platform
  10 import tempfile
  11 import logging
  12 import re
  13 import shutil
  14 import subprocess
  15
  16 try:
  17     import configparser
  18 except ImportError:
  19     import ConfigParser as configparser
  20
  21 def data_equal(a, b):
  22     # Allow multiple values in assignment separated by '|'
  23     a_list = a.split('|')
  24     b_list = b.split('|')
  25
  26     for a_item in a_list:
  27         for b_item in b_list:
  28             if (a_item == b_item):
  29                 return True
  30             elif (a_item == '*') or (b_item == '*'):
  31                 return True
  32
  33     return False
  34
  35 class Fail(Exception):
  36     def __init__(self, test, msg):
  37         self.msg = msg
  38         self.test = test
  39     def getMsg(self):
  40         return '\'%s\' - %s' % (self.test.path, self.msg)
  41
  42 class Notest(Exception):
  43     def __init__(self, test, arch):
  44         self.arch = arch
  45         self.test = test
  46     def getMsg(self):
  47         return '[%s] \'%s\'' % (self.arch, self.test.path)
  48
  49 class Unsup(Exception):
  50     def __init__(self, test):
  51         self.test = test
  52     def getMsg(self):
  53         return '\'%s\'' % self.test.path
  54
  55 class Event(dict):
  56     terms = [
  57         'cpu',
  58         'flags',
  59         'type',
  60         'size',
  61         'config',
  62         'sample_period',
  63         'sample_type',
  64         'read_format',
  65         'disabled',
  66         'inherit',
  67         'pinned',
  68         'exclusive',
  69         'exclude_user',
  70         'exclude_kernel',
  71         'exclude_hv',
  72         'exclude_idle',
  73         'mmap',
  74         'comm',
  75         'freq',
  76         'inherit_stat',
  77         'enable_on_exec',
  78         'task',
  79         'watermark',
  80         'precise_ip',
  81         'mmap_data',
  82         'sample_id_all',
  83         'exclude_host',
  84         'exclude_guest',
  85         'exclude_callchain_kernel',
  86         'exclude_callchain_user',
  87         'wakeup_events',
  88         'bp_type',
  89         'config1',
  90         'config2',
  91         'branch_sample_type',
  92         'sample_regs_user',
  93         'sample_stack_user',
  94     ]
  95
  96     def add(self, data):
  97         for key, val in data:
  98             log.debug("      %s = %s" % (key, val))
  99             self[key] = val
 100
 101     def __init__(self, name, data, base):
 102         log.debug("    Event %s" % name);
 103         self.name  = name;
 104         self.group = ''
 105         self.add(base)
 106         self.add(data)
 107
 108     def equal(self, other):
 109         for t in Event.terms:
 110             log.debug("      [%s] %s %s" % (t, self[t], other[t]));
 111             if t not in self or t not in other:
 112                 return False
 113             if not data_equal(self[t], other[t]):
 114                 return False
 115         return True
 116
 117     def optional(self):
 118         if 'optional' in self and self['optional'] == '1':
 119             return True
 120         return False
 121
 122     def diff(self, other):
 123         for t in Event.terms:
 124             if t not in self or t not in other:
 125                 continue
 126             if not data_equal(self[t], other[t]):
 127                 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
 128
 129 def parse_version(version):
 130     if not version:
 131         return None
 132     return [int(v) for v in version.split(".")[0:2]]
 133
 134 # Test file description needs to have following sections:
 135 # [config]
 136 #   - just single instance in file
 137 #   - needs to specify:
 138 #     'command' - perf command name
 139 #     'args'    - special command arguments
 140 #     'ret'     - Skip test if Perf doesn't exit with this value (0 by default)
 141 #     'test_ret'- If set to 'true', fail test instead of skipping for 'ret' argument
 142 #     'arch'    - architecture specific test (optional)
 143 #                 comma separated list, ! at the beginning
 144 #                 negates it.
 145 #     'auxv'    - Truthy statement that is evaled in the scope of the auxv map. When false,
 146 #                 the test is skipped. For example 'auxv["AT_HWCAP"] == 10'. (optional)
 147 #     'kernel_since' - Inclusive kernel version from which the test will start running. Only the
 148 #                      first two values are supported, for example "6.1" (optional)
 149 #     'kernel_until' - Exclusive kernel version from which the test will stop running. (optional)
 150 # [eventX:base]
 151 #   - one or multiple instances in file
 152 #   - expected values assignments
 153 class Test(object):
 154     def __init__(self, path, options):
 155         parser = configparser.ConfigParser()
 156         parser.read(path)
 157
 158         log.warning("running '%s'" % path)
 159
 160         self.path     = path
 161         self.test_dir = options.test_dir
 162         self.perf     = options.perf
 163         self.command  = parser.get('config', 'command')
 164         self.args     = parser.get('config', 'args')
 165
 166         try:
 167             self.ret  = parser.get('config', 'ret')
 168         except:
 169             self.ret  = 0
 170
 171         self.test_ret = parser.getboolean('config', 'test_ret', fallback=False)
 172
 173         try:
 174             self.arch  = parser.get('config', 'arch')
 175             log.warning("test limitation '%s'" % self.arch)
 176         except:
 177             self.arch  = ''
 178
 179         self.auxv = parser.get('config', 'auxv', fallback=None)
 180         self.kernel_since = parse_version(parser.get('config', 'kernel_since', fallback=None))
 181         self.kernel_until = parse_version(parser.get('config', 'kernel_until', fallback=None))
 182         self.expect   = {}
 183         self.result   = {}
 184         log.debug("  loading expected events");
 185         self.load_events(path, self.expect)
 186
 187     def is_event(self, name):
 188         if name.find("event") == -1:
 189             return False
 190         else:
 191             return True
 192
 193     def skip_test_kernel_since(self):
 194         if not self.kernel_since:
 195             return False
 196         return not self.kernel_since <= parse_version(platform.release())
 197
 198     def skip_test_kernel_until(self):
 199         if not self.kernel_until:
 200             return False
 201         return not parse_version(platform.release()) < self.kernel_until
 202
 203     def skip_test_auxv(self):
 204         def new_auxv(a, pattern):
 205             items = list(filter(None, pattern.split(a)))
 206             # AT_HWCAP is hex but doesn't have a prefix, so special case it
 207             if items[0] == "AT_HWCAP":
 208                 value = int(items[-1], 16)
 209             else:
 210                 try:
 211                     value = int(items[-1], 0)
 212                 except:
 213                     value = items[-1]
 214             return (items[0], value)
 215
 216         if not self.auxv:
 217             return False
 218         auxv = subprocess.check_output("LD_SHOW_AUXV=1 sleep 0", shell=True) \
 219                .decode(sys.stdout.encoding)
 220         pattern = re.compile(r"[: ]+")
 221         auxv = dict([new_auxv(a, pattern) for a in auxv.splitlines()])
 222         return not eval(self.auxv)
 223
 224     def skip_test_arch(self, myarch):
 225         # If architecture not set always run test
 226         if self.arch == '':
 227             # log.warning("test for arch %s is ok" % myarch)
 228             return False
 229
 230         # Allow multiple values in assignment separated by ','
 231         arch_list = self.arch.split(',')
 232
 233         # Handle negated list such as !s390x,ppc
 234         if arch_list[0][0] == '!':
 235             arch_list[0] = arch_list[0][1:]
 236             log.warning("excluded architecture list %s" % arch_list)
 237             for arch_item in arch_list:
 238                 # log.warning("test for %s arch is %s" % (arch_item, myarch))
 239                 if arch_item == myarch:
 240                     return True
 241             return False
 242
 243         for arch_item in arch_list:
 244             # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
 245             if arch_item == myarch:
 246                 return False
 247         return True
 248
 249     def restore_sample_rate(self, value=10000):
 250         try:
 251             # Check value of sample_rate
 252             with open("/proc/sys/kernel/perf_event_max_sample_rate", "r") as fIn:
 253                 curr_value = fIn.readline()
 254             # If too low restore to reasonable value
 255             if not curr_value or int(curr_value) < int(value):
 256                 with open("/proc/sys/kernel/perf_event_max_sample_rate", "w") as fOut:
 257                     fOut.write(str(value))
 258
 259         except IOError as e:
 260             log.warning("couldn't restore sample_rate value: I/O error %s" % e)
 261         except ValueError as e:
 262             log.warning("couldn't restore sample_rate value: Value error %s" % e)
 263         except TypeError as e:
 264             log.warning("couldn't restore sample_rate value: Type error %s" % e)
 265
 266     def load_events(self, path, events):
 267         parser_event = configparser.ConfigParser()
 268         parser_event.read(path)
 269
 270         # The event record section header contains 'event' word,
 271         # optionaly followed by ':' allowing to load 'parent
 272         # event' first as a base
 273         for section in filter(self.is_event, parser_event.sections()):
 274
 275             parser_items = parser_event.items(section);
 276             base_items   = {}
 277
 278             # Read parent event if there's any
 279             if (':' in section):
 280                 base = section[section.index(':') + 1:]
 281                 parser_base = configparser.ConfigParser()
 282                 parser_base.read(self.test_dir + '/' + base)
 283                 base_items = parser_base.items('event')
 284
 285             e = Event(section, parser_items, base_items)
 286             events[section] = e
 287
 288     def run_cmd(self, tempdir):
 289         junk1, junk2, junk3, junk4, myarch = (os.uname())
 290
 291         if self.skip_test_arch(myarch):
 292             raise Notest(self, myarch)
 293
 294         if self.skip_test_auxv():
 295             raise Notest(self, "auxv skip")
 296
 297         if self.skip_test_kernel_since():
 298             raise Notest(self, "old kernel skip")
 299
 300         if self.skip_test_kernel_until():
 301             raise Notest(self, "new kernel skip")
 302
 303         self.restore_sample_rate()
 304         cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
 305               self.perf, self.command, tempdir, self.args)
 306         ret = os.WEXITSTATUS(os.system(cmd))
 307
 308         log.info("  '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
 309
 310         if not data_equal(str(ret), str(self.ret)):
 311             if self.test_ret:
 312                 raise Fail(self, "Perf exit code failure")
 313             else:
 314                 raise Unsup(self)
 315
 316     def compare(self, expect, result):
 317         match = {}
 318
 319         log.debug("  compare");
 320
 321         # For each expected event find all matching
 322         # events in result. Fail if there's not any.
 323         for exp_name, exp_event in expect.items():
 324             exp_list = []
 325             res_event = {}
 326             log.debug("    matching [%s]" % exp_name)
 327             for res_name, res_event in result.items():
 328                 log.debug("      to [%s]" % res_name)
 329                 if (exp_event.equal(res_event)):
 330                     exp_list.append(res_name)
 331                     log.debug("    ->OK")
 332                 else:
 333                     log.debug("    ->FAIL");
 334
 335             log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
 336
 337             # we did not any matching event - fail
 338             if not exp_list:
 339                 if exp_event.optional():
 340                     log.debug("    %s does not match, but is optional" % exp_name)
 341                 else:
 342                     if not res_event:
 343                         log.debug("    res_event is empty");
 344                     else:
 345                         exp_event.diff(res_event)
 346                     raise Fail(self, 'match failure');
 347
 348             match[exp_name] = exp_list
 349
 350         # For each defined group in the expected events
 351         # check we match the same group in the result.
 352         for exp_name, exp_event in expect.items():
 353             group = exp_event.group
 354
 355             if (group == ''):
 356                 continue
 357
 358             for res_name in match[exp_name]:
 359                 res_group = result[res_name].group
 360                 if res_group not in match[group]:
 361                     raise Fail(self, 'group failure')
 362
 363                 log.debug("    group: [%s] matches group leader %s" %
 364                          (exp_name, str(match[group])))
 365
 366         log.debug("  matched")
 367
 368     def resolve_groups(self, events):
 369         for name, event in events.items():
 370             group_fd = event['group_fd'];
 371             if group_fd == '-1':
 372                 continue;
 373
 374             for iname, ievent in events.items():
 375                 if (ievent['fd'] == group_fd):
 376                     event.group = iname
 377                     log.debug('[%s] has group leader [%s]' % (name, iname))
 378                     break;
 379
 380     def run(self):
 381         tempdir = tempfile.mkdtemp();
 382
 383         try:
 384             # run the test script
 385             self.run_cmd(tempdir);
 386
 387             # load events expectation for the test
 388             log.debug("  loading result events");
 389             for f in glob.glob(tempdir + '/event*'):
 390                 self.load_events(f, self.result);
 391
 392             # resolve group_fd to event names
 393             self.resolve_groups(self.expect);
 394             self.resolve_groups(self.result);
 395
 396             # do the expectation - results matching - both ways
 397             self.compare(self.expect, self.result)
 398             self.compare(self.result, self.expect)
 399
 400         finally:
 401             # cleanup
 402             shutil.rmtree(tempdir)
 403
 404
 405 def run_tests(options):
 406     for f in glob.glob(options.test_dir + '/' + options.test):
 407         try:
 408             Test(f, options).run()
 409         except Unsup as obj:
 410             log.warning("unsupp  %s" % obj.getMsg())
 411         except Notest as obj:
 412             log.warning("skipped %s" % obj.getMsg())
 413
 414 def setup_log(verbose):
 415     global log
 416     level = logging.CRITICAL
 417
 418     if verbose == 1:
 419         level = logging.WARNING
 420     if verbose == 2:
 421         level = logging.INFO
 422     if verbose >= 3:
 423         level = logging.DEBUG
 424
 425     log = logging.getLogger('test')
 426     log.setLevel(level)
 427     ch  = logging.StreamHandler()
 428     ch.setLevel(level)
 429     formatter = logging.Formatter('%(message)s')
 430     ch.setFormatter(formatter)
 431     log.addHandler(ch)
 432
 433 USAGE = '''%s [OPTIONS]
 434   -d dir  # tests dir
 435   -p path # perf binary
 436   -t test # single test
 437   -v      # verbose level
 438 ''' % sys.argv[0]
 439
 440 def main():
 441     parser = optparse.OptionParser(usage=USAGE)
 442
 443     parser.add_option("-t", "--test",
 444                       action="store", type="string", dest="test")
 445     parser.add_option("-d", "--test-dir",
 446                       action="store", type="string", dest="test_dir")
 447     parser.add_option("-p", "--perf",
 448                       action="store", type="string", dest="perf")
 449     parser.add_option("-v", "--verbose",
 450                       default=0, action="count", dest="verbose")
 451
 452     options, args = parser.parse_args()
 453     if args:
 454         parser.error('FAILED wrong arguments %s' %  ' '.join(args))
 455         return -1
 456
 457     setup_log(options.verbose)
 458
 459     if not options.test_dir:
 460         print('FAILED no -d option specified')
 461         sys.exit(-1)
 462
 463     if not options.test:
 464         options.test = 'test*'
 465
 466     try:
 467         run_tests(options)
 468
 469     except Fail as obj:
 470         print("FAILED %s" % obj.getMsg())
 471         sys.exit(-1)
 472
 473     sys.exit(0)
 474
 475 if __name__ == '__main__':
 476     main()