tools/perf/tests/attr.py

   1 # SPDX-License-Identifier: GPL-2.0
   2
   3 from __future__ import print_function
   4
   5 import os
   6 import sys
   7 import glob
   8 import optparse
   9 import tempfile
  10 import logging
  11 import shutil
  12
  13 try:
  14     import configparser
  15 except ImportError:
  16     import ConfigParser as configparser
  17
  18 def data_equal(a, b):
  19     # Allow multiple values in assignment separated by '|'
  20     a_list = a.split('|')
  21     b_list = b.split('|')
  22
  23     for a_item in a_list:
  24         for b_item in b_list:
  25             if (a_item == b_item):
  26                 return True
  27             elif (a_item == '*') or (b_item == '*'):
  28                 return True
  29
  30     return False
  31
  32 class Fail(Exception):
  33     def __init__(self, test, msg):
  34         self.msg = msg
  35         self.test = test
  36     def getMsg(self):
  37         return '\'%s\' - %s' % (self.test.path, self.msg)
  38
  39 class Notest(Exception):
  40     def __init__(self, test, arch):
  41         self.arch = arch
  42         self.test = test
  43     def getMsg(self):
  44         return '[%s] \'%s\'' % (self.arch, self.test.path)
  45
  46 class Unsup(Exception):
  47     def __init__(self, test):
  48         self.test = test
  49     def getMsg(self):
  50         return '\'%s\'' % self.test.path
  51
  52 class Event(dict):
  53     terms = [
  54         'cpu',
  55         'flags',
  56         'type',
  57         'size',
  58         'config',
  59         'sample_period',
  60         'sample_type',
  61         'read_format',
  62         'disabled',
  63         'inherit',
  64         'pinned',
  65         'exclusive',
  66         'exclude_user',
  67         'exclude_kernel',
  68         'exclude_hv',
  69         'exclude_idle',
  70         'mmap',
  71         'comm',
  72         'freq',
  73         'inherit_stat',
  74         'enable_on_exec',
  75         'task',
  76         'watermark',
  77         'precise_ip',
  78         'mmap_data',
  79         'sample_id_all',
  80         'exclude_host',
  81         'exclude_guest',
  82         'exclude_callchain_kernel',
  83         'exclude_callchain_user',
  84         'wakeup_events',
  85         'bp_type',
  86         'config1',
  87         'config2',
  88         'branch_sample_type',
  89         'sample_regs_user',
  90         'sample_stack_user',
  91     ]
  92
  93     def add(self, data):
  94         for key, val in data:
  95             log.debug("      %s = %s" % (key, val))
  96             self[key] = val
  97
  98     def __init__(self, name, data, base):
  99         log.debug("    Event %s" % name);
 100         self.name  = name;
 101         self.group = ''
 102         self.add(base)
 103         self.add(data)
 104
 105     def equal(self, other):
 106         for t in Event.terms:
 107             log.debug("      [%s] %s %s" % (t, self[t], other[t]));
 108             if t not in self or t not in other:
 109                 return False
 110             if not data_equal(self[t], other[t]):
 111                 return False
 112         return True
 113
 114     def optional(self):
 115         if 'optional' in self and self['optional'] == '1':
 116             return True
 117         return False
 118
 119     def diff(self, other):
 120         for t in Event.terms:
 121             if t not in self or t not in other:
 122                 continue
 123             if not data_equal(self[t], other[t]):
 124                 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
 125
 126 # Test file description needs to have following sections:
 127 # [config]
 128 #   - just single instance in file
 129 #   - needs to specify:
 130 #     'command' - perf command name
 131 #     'args'    - special command arguments
 132 #     'ret'     - expected command return value (0 by default)
 133 #     'arch'    - architecture specific test (optional)
 134 #                 comma separated list, ! at the beginning
 135 #                 negates it.
 136 #
 137 # [eventX:base]
 138 #   - one or multiple instances in file
 139 #   - expected values assignments
 140 class Test(object):
 141     def __init__(self, path, options):
 142         parser = configparser.SafeConfigParser()
 143         parser.read(path)
 144
 145         log.warning("running '%s'" % path)
 146
 147         self.path     = path
 148         self.test_dir = options.test_dir
 149         self.perf     = options.perf
 150         self.command  = parser.get('config', 'command')
 151         self.args     = parser.get('config', 'args')
 152
 153         try:
 154             self.ret  = parser.get('config', 'ret')
 155         except:
 156             self.ret  = 0
 157
 158         try:
 159             self.arch  = parser.get('config', 'arch')
 160             log.warning("test limitation '%s'" % self.arch)
 161         except:
 162             self.arch  = ''
 163
 164         self.expect   = {}
 165         self.result   = {}
 166         log.debug("  loading expected events");
 167         self.load_events(path, self.expect)
 168
 169     def is_event(self, name):
 170         if name.find("event") == -1:
 171             return False
 172         else:
 173             return True
 174
 175     def skip_test(self, myarch):
 176         # If architecture not set always run test
 177         if self.arch == '':
 178             # log.warning("test for arch %s is ok" % myarch)
 179             return False
 180
 181         # Allow multiple values in assignment separated by ','
 182         arch_list = self.arch.split(',')
 183
 184         # Handle negated list such as !s390x,ppc
 185         if arch_list[0][0] == '!':
 186             arch_list[0] = arch_list[0][1:]
 187             log.warning("excluded architecture list %s" % arch_list)
 188             for arch_item in arch_list:
 189                 # log.warning("test for %s arch is %s" % (arch_item, myarch))
 190                 if arch_item == myarch:
 191                     return True
 192             return False
 193
 194         for arch_item in arch_list:
 195             # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
 196             if arch_item == myarch:
 197                 return False
 198         return True
 199
 200     def load_events(self, path, events):
 201         parser_event = configparser.SafeConfigParser()
 202         parser_event.read(path)
 203
 204         # The event record section header contains 'event' word,
 205         # optionaly followed by ':' allowing to load 'parent
 206         # event' first as a base
 207         for section in filter(self.is_event, parser_event.sections()):
 208
 209             parser_items = parser_event.items(section);
 210             base_items   = {}
 211
 212             # Read parent event if there's any
 213             if (':' in section):
 214                 base = section[section.index(':') + 1:]
 215                 parser_base = configparser.SafeConfigParser()
 216                 parser_base.read(self.test_dir + '/' + base)
 217                 base_items = parser_base.items('event')
 218
 219             e = Event(section, parser_items, base_items)
 220             events[section] = e
 221
 222     def run_cmd(self, tempdir):
 223         junk1, junk2, junk3, junk4, myarch = (os.uname())
 224
 225         if self.skip_test(myarch):
 226             raise Notest(self, myarch)
 227
 228         cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
 229               self.perf, self.command, tempdir, self.args)
 230         ret = os.WEXITSTATUS(os.system(cmd))
 231
 232         log.info("  '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
 233
 234         if not data_equal(str(ret), str(self.ret)):
 235             raise Unsup(self)
 236
 237     def compare(self, expect, result):
 238         match = {}
 239
 240         log.debug("  compare");
 241
 242         # For each expected event find all matching
 243         # events in result. Fail if there's not any.
 244         for exp_name, exp_event in expect.items():
 245             exp_list = []
 246             res_event = {}
 247             log.debug("    matching [%s]" % exp_name)
 248             for res_name, res_event in result.items():
 249                 log.debug("      to [%s]" % res_name)
 250                 if (exp_event.equal(res_event)):
 251                     exp_list.append(res_name)
 252                     log.debug("    ->OK")
 253                 else:
 254                     log.debug("    ->FAIL");
 255
 256             log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
 257
 258             # we did not any matching event - fail
 259             if not exp_list:
 260                 if exp_event.optional():
 261                     log.debug("    %s does not match, but is optional" % exp_name)
 262                 else:
 263                     if not res_event:
 264                         log.debug("    res_event is empty");
 265                     else:
 266                         exp_event.diff(res_event)
 267                     raise Fail(self, 'match failure');
 268
 269             match[exp_name] = exp_list
 270
 271         # For each defined group in the expected events
 272         # check we match the same group in the result.
 273         for exp_name, exp_event in expect.items():
 274             group = exp_event.group
 275
 276             if (group == ''):
 277                 continue
 278
 279             for res_name in match[exp_name]:
 280                 res_group = result[res_name].group
 281                 if res_group not in match[group]:
 282                     raise Fail(self, 'group failure')
 283
 284                 log.debug("    group: [%s] matches group leader %s" %
 285                          (exp_name, str(match[group])))
 286
 287         log.debug("  matched")
 288
 289     def resolve_groups(self, events):
 290         for name, event in events.items():
 291             group_fd = event['group_fd'];
 292             if group_fd == '-1':
 293                 continue;
 294
 295             for iname, ievent in events.items():
 296                 if (ievent['fd'] == group_fd):
 297                     event.group = iname
 298                     log.debug('[%s] has group leader [%s]' % (name, iname))
 299                     break;
 300
 301     def run(self):
 302         tempdir = tempfile.mkdtemp();
 303
 304         try:
 305             # run the test script
 306             self.run_cmd(tempdir);
 307
 308             # load events expectation for the test
 309             log.debug("  loading result events");
 310             for f in glob.glob(tempdir + '/event*'):
 311                 self.load_events(f, self.result);
 312
 313             # resolve group_fd to event names
 314             self.resolve_groups(self.expect);
 315             self.resolve_groups(self.result);
 316
 317             # do the expectation - results matching - both ways
 318             self.compare(self.expect, self.result)
 319             self.compare(self.result, self.expect)
 320
 321         finally:
 322             # cleanup
 323             shutil.rmtree(tempdir)
 324
 325
 326 def run_tests(options):
 327     for f in glob.glob(options.test_dir + '/' + options.test):
 328         try:
 329             Test(f, options).run()
 330         except Unsup as obj:
 331             log.warning("unsupp  %s" % obj.getMsg())
 332         except Notest as obj:
 333             log.warning("skipped %s" % obj.getMsg())
 334
 335 def setup_log(verbose):
 336     global log
 337     level = logging.CRITICAL
 338
 339     if verbose == 1:
 340         level = logging.WARNING
 341     if verbose == 2:
 342         level = logging.INFO
 343     if verbose >= 3:
 344         level = logging.DEBUG
 345
 346     log = logging.getLogger('test')
 347     log.setLevel(level)
 348     ch  = logging.StreamHandler()
 349     ch.setLevel(level)
 350     formatter = logging.Formatter('%(message)s')
 351     ch.setFormatter(formatter)
 352     log.addHandler(ch)
 353
 354 USAGE = '''%s [OPTIONS]
 355   -d dir  # tests dir
 356   -p path # perf binary
 357   -t test # single test
 358   -v      # verbose level
 359 ''' % sys.argv[0]
 360
 361 def main():
 362     parser = optparse.OptionParser(usage=USAGE)
 363
 364     parser.add_option("-t", "--test",
 365                       action="store", type="string", dest="test")
 366     parser.add_option("-d", "--test-dir",
 367                       action="store", type="string", dest="test_dir")
 368     parser.add_option("-p", "--perf",
 369                       action="store", type="string", dest="perf")
 370     parser.add_option("-v", "--verbose",
 371                       default=0, action="count", dest="verbose")
 372
 373     options, args = parser.parse_args()
 374     if args:
 375         parser.error('FAILED wrong arguments %s' %  ' '.join(args))
 376         return -1
 377
 378     setup_log(options.verbose)
 379
 380     if not options.test_dir:
 381         print('FAILED no -d option specified')
 382         sys.exit(-1)
 383
 384     if not options.test:
 385         options.test = 'test*'
 386
 387     try:
 388         run_tests(options)
 389
 390     except Fail as obj:
 391         print("FAILED %s" % obj.getMsg())
 392         sys.exit(-1)
 393
 394     sys.exit(0)
 395
 396 if __name__ == '__main__':
 397     main()