framework/monitoring.py

   1 # coding=utf-8
   2 # Copyright (c) 2016, 2019 Intel Corporation
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the "Software"),
   6 # to deal in the Software without restriction, including without limitation
   7 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 # and/or sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following conditions:
  10 #
  11 # The above copyright notice and this permission notice (including the next
  12 # paragraph) shall be included in all copies or substantial portions of the
  13 # Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  18 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22
  23 """Module implementing classes for monitoring
  24
  25 This module provides a set of classes for monitoring the system on multiple
  26 sources. The sources can be standard files, locked files or the dmesg.
  27 The patterns of the errors are defined in piglit.conf, at the section
  28 monitored-errors.
  29
  30 When one of the regex is found in the corresponding source Piglit will abort
  31 with code 3.
  32
  33 """
  34
  35 import abc
  36 import errno
  37 import os
  38 import re
  39
  40 from framework.core import PIGLIT_CONFIG
  41 from framework.dmesg import LinuxDmesg
  42 from framework import exceptions
  43
  44 __all__ = [
  45     'BaseMonitoring',
  46     'Monitoring',
  47     'MonitoringFile',
  48     'MonitoringLinuxDmesg',
  49 ]
  50
  51
  52
  53 class Monitoring(object):
  54     """Class that initialize and update monitoring classes
  55
  56     This reads the rules from the section monitored-errors in piglit.conf
  57     and initializes the monitoring objects. Their type must be derived from
  58     BaseMonitoring and specialized according the field 'type'.
  59
  60     """
  61
  62     # starting time: user must know current machine/GPU state before
  63     # starting piglit: we can resume test execution based on last stopping
  64     # if we consider that our system is still in good shape (ie not blocking
  65     # run forever)
  66     _abort_error = None
  67     _monitoring_rules = None
  68
  69     def __init__(self, monitoring_enabled):
  70         """Create a LinuxMonitored instance"""
  71         # Get the monitoring rules from piglit.conf and store them into a dict.
  72         self._monitoring_rules = {}
  73
  74         if monitoring_enabled and PIGLIT_CONFIG.has_section('monitored-errors'):
  75             for key, _ in PIGLIT_CONFIG.items('monitored-errors'):
  76                 if PIGLIT_CONFIG.has_section(key):
  77                     type = PIGLIT_CONFIG.required_get(key, 'type')
  78                     regex = PIGLIT_CONFIG.required_get(key, 'regex')
  79                     parameters = PIGLIT_CONFIG.required_get(key, 'parameters')
  80
  81                     self.add_rule(key, type, parameters, regex)
  82
  83     @property
  84     def abort_needed(self):
  85         """Simply return if _abort_error variable is not empty"""
  86         return self._abort_error is not None
  87
  88     @property
  89     def error_message(self):
  90         """Simply return _abort_error message"""
  91         return self._abort_error
  92
  93     def add_rule(self, key, type, parameters, regex):
  94         """Add a new monitoring rule
  95
  96         This method adds a new monitoring rule. The type must be file,
  97         locked_file or dmesg.
  98
  99         Arguments:
 100         key -- The key for storing the rule in a dict
 101         type -- The type of monitoring
 102         parameters -- Depending on the type, can be a file path or a command
 103                       options
 104         regex -- The rule regex
 105
 106         """
 107         rule = None
 108
 109         if type == 'file':
 110             rule = MonitoringFile(parameters,
 111                                   regex)
 112         elif type == 'locked_file':
 113             rule = MonitoringFile(parameters,
 114                                   regex, True)
 115         elif type == 'dmesg':
 116             rule = MonitoringLinuxDmesg(parameters,
 117                                         regex)
 118         else:
 119             raise exceptions.PiglitFatalError(
 120                 "No available monitoring class for the type {}.".format(type))
 121
 122         self._monitoring_rules[key] = rule
 123
 124     def delete_rule(self, key):
 125         """Remove a monitoring rule
 126
 127         Arguments:
 128         key -- The rule key
 129
 130         """
 131         self._monitoring_rules.pop(key, None)
 132
 133     def update_monitoring(self):
 134         """Update the new messages for each monitoring object"""
 135         if self._monitoring_rules:
 136             for monitoring_rule in self._monitoring_rules.values():
 137                 monitoring_rule.update_monitoring()
 138
 139     def check_monitoring(self):
 140         """Check monitoring objects statue
 141
 142         This method checks the state for each monitoring object.
 143         If one of them found the pattern in the new messages,
 144         set itself on abort_needed state.
 145
 146         """
 147         # Get a new snapshot of the source
 148         self.update_monitoring()
 149
 150         if self._monitoring_rules:
 151             for rule_key, monitoring_rule in self._monitoring_rules.items():
 152                 # Get error message
 153                 self._abort_error = monitoring_rule.check_monitoring()
 154                 # if error message is not empty, abort is requested
 155                 if self.abort_needed:
 156                     self._abort_error = "From the rule {}:\n{}".format(
 157                         rule_key,
 158                         self._abort_error)
 159                     break
 160
 161
 162 class BaseMonitoring(metaclass=abc.ABCMeta):
 163     """Abstract base class for Monitoring derived objects
 164
 165     This provides the bases of the constructor and most subclasses should call
 166     super() in __init__(). It provides a concrete implementation of the
 167     check_monitoring() method, which should be suitible for all subclasses.
 168
 169     The update_monitoring() method need to be override for all subclasses.
 170
 171     """
 172     @abc.abstractmethod
 173     def __init__(self, monitoring_source, regex):
 174         """Abstract constructor for BaseMonitoring subclasses
 175
 176         Arguments;
 177         monitoring_source -- The source to monitor
 178         regex -- The errors pattern
 179
 180         """
 181         self._monitoring_source = monitoring_source
 182         self._monitoring_regex = re.compile(regex)
 183         self._new_messages = ''
 184
 185     @property
 186     def new_messages(self):
 187         """Simply return if _abort_error variable is not empty"""
 188         return self._new_messages
 189
 190     @abc.abstractmethod
 191     def update_monitoring(self):
 192         """Update _new_messages list
 193
 194         This method should read a monitoring source like a file or a program
 195         output, determine new entries since the last read of the source,
 196         and update self._new_messages with those entries
 197
 198         """
 199         pass
 200
 201     def check_monitoring(self):
 202         """Check _new_messages
 203
 204         This method checks if the regex is found in the new messages,
 205         then return a string that contain the monitored location and
 206         the matched line
 207
 208         """
 209         if self._new_messages and self._monitoring_regex:
 210             for line in self._new_messages:
 211                 if self._monitoring_regex.search(line):
 212                     return line
 213
 214
 215 if os.name == 'posix':
 216     import fcntl
 217
 218
 219     class MonitoringFile(BaseMonitoring):
 220         """Monitoring from a file
 221
 222         This class is for monitoring the system from a file that
 223         can be a standard file or a locked file. The locked file
 224         algorithm uses fnctl, so it doesn't work on non Unix systems
 225
 226         Arguments:
 227         is_locked -- True if the target is a locked file
 228
 229         """
 230         _is_locked = False
 231
 232         def __init__(self, monitoring_source, regex, is_locked=False):
 233             """Create a MonitoringFile instance"""
 234             self._last_message = None
 235             self._is_locked = is_locked
 236             super(MonitoringFile, self).__init__(monitoring_source, regex)
 237
 238         def update_monitoring(self):
 239             """Open the file and get the differences
 240
 241             Get the contents of the file, then calculate new messages.
 242             This implements also a specific method for reading locked files.
 243
 244             """
 245
 246             try:
 247                 with open(self._monitoring_source, 'r') as f:
 248                     lines = []
 249                     if self._is_locked:
 250                         # Create a duplicated file descriptor, this avoid lock
 251                         fd = os.dup(f.fileno())
 252                         # use I/O control for reading the lines
 253                         fcntl.fcntl(fd, fcntl.F_SETFL, os.O_NONBLOCK)
 254
 255                         while True:
 256                             try:
 257                                 line = os.read(fd, 1024)
 258                                 if not line:
 259                                     break;
 260                             except OSError as e:
 261                                 if e.errno == errno.EAGAIN:
 262                                     break
 263                                 else:
 264                                     raise e
 265                             lines.append(line.decode("utf-8", "strict"))
 266                         os.close(fd)
 267
 268                     else:
 269                         lines = f.read().splitlines()
 270
 271                     f.close()
 272
 273                     # Find all new entries, do this by slicing the list of the
 274                     # lines to only returns elements after the last element
 275                     # stored. If there are not matches a value error is raised,
 276                     # that means all of the lines are new
 277                     l = 0
 278                     for index, item in enumerate(reversed(lines)):
 279                         if item == self._last_message:
 280                             l = len(lines) - index  # don't include the matched element
 281                             break
 282                     self._new_messages = lines[l:]
 283                     # Attempt to store the last element of lines,
 284                     # unless there was no line
 285                     self._last_message = lines[-1] if lines else None
 286             except Exception:
 287                 # if an error occurred, we consider there are no new messages
 288                 self._new_messages = []
 289                 pass
 290
 291
 292     class MonitoringLinuxDmesg(BaseMonitoring, LinuxDmesg):
 293         """Monitoring on dmesg
 294
 295         This class is for monitoring on the system dmesg. It's inherited
 296         from LinuxDmesg for the dmesg processing methods.
 297
 298         Work only on Linux operating system.
 299
 300         """
 301         def __init__(self, monitoring_source, regex):
 302             """Create a MonitoringLinuxDmesg instance"""
 303             self.DMESG_COMMAND = ['dmesg']+monitoring_source.split()
 304             BaseMonitoring.__init__(self, monitoring_source, regex)
 305             LinuxDmesg.__init__(self)
 306
 307         def update_monitoring(self):
 308             """Call update_dmesg"""
 309             self.update_dmesg()