tools/testing/selftests/powerpc/eeh/eeh-functions.sh

   1 #!/bin/sh
   2 # SPDX-License-Identifier: GPL-2.0-only
   3
   4 export KSELFTESTS_SKIP=4
   5
   6 log() {
   7         echo >/dev/stderr $*
   8 }
   9
  10 pe_ok() {
  11         local dev="$1"
  12         local path="/sys/bus/pci/devices/$dev/eeh_pe_state"
  13
  14         # if a driver doesn't support the error handling callbacks then the
  15         # device is recovered by removing and re-probing it. This causes the
  16         # sysfs directory to disappear so read the PE state once and squash
  17         # any potential error messages
  18         local eeh_state="$(cat $path 2>/dev/null)"
  19         if [ -z "$eeh_state" ]; then
  20                 return 1;
  21         fi
  22
  23         local fw_state="$(echo $eeh_state | cut -d' ' -f1)"
  24         local sw_state="$(echo $eeh_state | cut -d' ' -f2)"
  25
  26         # If EEH_PE_ISOLATED or EEH_PE_RECOVERING are set then the PE is in an
  27         # error state or being recovered. Either way, not ok.
  28         if [ "$((sw_state & 0x3))" -ne 0 ] ; then
  29                 return 1
  30         fi
  31
  32         # A functioning PE should have the EEH_STATE_MMIO_ACTIVE and
  33         # EEH_STATE_DMA_ACTIVE flags set. For some goddamn stupid reason
  34         # the platform backends set these when the PE is in reset. The
  35         # RECOVERING check above should stop any false positives though.
  36         if [ "$((fw_state & 0x18))" -ne "$((0x18))" ] ; then
  37                 return 1
  38         fi
  39
  40         return 0;
  41 }
  42
  43 eeh_supported() {
  44         test -e /proc/powerpc/eeh && \
  45         grep -q 'EEH Subsystem is enabled' /proc/powerpc/eeh
  46 }
  47
  48 eeh_test_prep() {
  49         if ! eeh_supported ; then
  50                 echo "EEH not supported on this system, skipping"
  51                 exit $KSELFTESTS_SKIP;
  52         fi
  53
  54         if [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_check" ] && \
  55            [ ! -e "/sys/kernel/debug/powerpc/eeh_dev_break" ] ; then
  56                 log "debugfs EEH testing files are missing. Is debugfs mounted?"
  57                 exit $KSELFTESTS_SKIP;
  58         fi
  59
  60         # Bump the max freeze count to something absurd so we don't
  61         # trip over it while breaking things.
  62         echo 5000 > /sys/kernel/debug/powerpc/eeh_max_freezes
  63 }
  64
  65 eeh_can_break() {
  66         # skip bridges since we can't recover them (yet...)
  67         if [ -e "/sys/bus/pci/devices/$dev/pci_bus" ] ; then
  68                 log "$dev, Skipped: bridge"
  69                 return 1;
  70         fi
  71
  72         # The ahci driver doesn't support error recovery. If the ahci device
  73         # happens to be hosting the root filesystem, and then we go and break
  74         # it the system will generally go down. We should probably fix that
  75         # at some point
  76         if [ "ahci" = "$(basename $(realpath /sys/bus/pci/devices/$dev/driver))" ] ; then
  77                 log "$dev, Skipped: ahci doesn't support recovery"
  78                 return 1;
  79         fi
  80
  81         # Don't inject errosr into an already-frozen PE. This happens with
  82         # PEs that contain multiple PCI devices (e.g. multi-function cards)
  83         # and injecting new errors during the recovery process will probably
  84         # result in the recovery failing and the device being marked as
  85         # failed.
  86         if ! pe_ok $dev ; then
  87                 log "$dev, Skipped: Bad initial PE state"
  88                 return 1;
  89         fi
  90
  91         return 0
  92 }
  93
  94 eeh_one_dev() {
  95         local dev="$1"
  96
  97         # Using this function from the command line is sometimes useful for
  98         # testing so check that the argument is a well-formed sysfs device
  99         # name.
 100         if ! test -e /sys/bus/pci/devices/$dev/ ; then
 101                 log "Error: '$dev' must be a sysfs device name (DDDD:BB:DD.F)"
 102                 return 1;
 103         fi
 104
 105         # Break it
 106         echo $dev >/sys/kernel/debug/powerpc/eeh_dev_break
 107
 108         # Force an EEH device check. If the kernel has already
 109         # noticed the EEH (due to a driver poll or whatever), this
 110         # is a no-op.
 111         echo $dev >/sys/kernel/debug/powerpc/eeh_dev_check
 112
 113         # Default to a 60s timeout when waiting for a device to recover. This
 114         # is an arbitrary default which can be overridden by setting the
 115         # EEH_MAX_WAIT environmental variable when required.
 116
 117         # The current record holder for longest recovery time is:
 118         #  "Adaptec Series 8 12G SAS/PCIe 3" at 39 seconds
 119         max_wait=${EEH_MAX_WAIT:=60}
 120
 121         for i in `seq 0 ${max_wait}` ; do
 122                 if pe_ok $dev ; then
 123                         break;
 124                 fi
 125                 log "$dev, waited $i/${max_wait}"
 126                 sleep 1
 127         done
 128
 129         if ! pe_ok $dev ; then
 130                 log "$dev, Failed to recover!"
 131                 return 1;
 132         fi
 133
 134         log "$dev, Recovered after $i seconds"
 135         return 0;
 136 }
 137
 138 eeh_has_driver() {
 139         test -e /sys/bus/pci/devices/$1/driver;
 140         return $?
 141 }
 142
 143 eeh_can_recover() {
 144         # we'll get an IO error if the device's current driver doesn't support
 145         # error recovery
 146         echo $1 > '/sys/kernel/debug/powerpc/eeh_dev_can_recover' 2>/dev/null
 147
 148         return $?
 149 }
 150
 151 eeh_find_all_pfs() {
 152         devices=""
 153
 154         # SR-IOV on pseries requires hypervisor support, so check for that
 155         is_pseries=""
 156         if grep -q pSeries /proc/cpuinfo ; then
 157                 if [ ! -f /proc/device-tree/rtas/ibm,open-sriov-allow-unfreeze ] ||
 158                    [ ! -f /proc/device-tree/rtas/ibm,open-sriov-map-pe-number ] ; then
 159                         return 1;
 160                 fi
 161
 162                 is_pseries="true"
 163         fi
 164
 165         for dev in `ls -1 /sys/bus/pci/devices/` ; do
 166                 sysfs="/sys/bus/pci/devices/$dev"
 167                 if [ ! -e "$sysfs/sriov_numvfs" ] ; then
 168                         continue
 169                 fi
 170
 171                 # skip unsupported PFs on pseries
 172                 if [ -z "$is_pseries" ] &&
 173                    [ ! -f "$sysfs/of_node/ibm,is-open-sriov-pf" ] &&
 174                    [ ! -f "$sysfs/of_node/ibm,open-sriov-vf-bar-info" ] ; then
 175                         continue;
 176                 fi
 177
 178                 # no driver, no vfs
 179                 if ! eeh_has_driver $dev ; then
 180                         continue
 181                 fi
 182
 183                 devices="$devices $dev"
 184         done
 185
 186         if [ -z "$devices" ] ; then
 187                 return 1;
 188         fi
 189
 190         echo $devices
 191         return 0;
 192 }
 193
 194 # attempts to enable one VF on each PF so we can do VF specific tests.
 195 # stdout: list of enabled VFs, one per line
 196 # return code: 0 if vfs are found, 1 otherwise
 197 eeh_enable_vfs() {
 198         pf_list="$(eeh_find_all_pfs)"
 199
 200         vfs=0
 201         for dev in $pf_list ; do
 202                 pf_sysfs="/sys/bus/pci/devices/$dev"
 203
 204                 # make sure we have a single VF
 205                 echo 0 > "$pf_sysfs/sriov_numvfs"
 206                 echo 1 > "$pf_sysfs/sriov_numvfs"
 207                 if [ "$?" != 0 ] ; then
 208                         log "Unable to enable VFs on $pf, skipping"
 209                         continue;
 210                 fi
 211
 212                 vf="$(basename $(realpath "$pf_sysfs/virtfn0"))"
 213                 if [ $? != 0 ] ; then
 214                         log "unable to find enabled vf on $pf"
 215                         echo 0 > "$pf_sysfs/sriov_numvfs"
 216                         continue;
 217                 fi
 218
 219                 if ! eeh_can_break $vf ; then
 220                         log "skipping "
 221
 222                         echo 0 > "$pf_sysfs/sriov_numvfs"
 223                         continue;
 224                 fi
 225
 226                 vfs="$((vfs + 1))"
 227                 echo $vf
 228         done
 229
 230         test "$vfs" != 0
 231         return $?
 232 }
 233
 234 eeh_disable_vfs() {
 235         pf_list="$(eeh_find_all_pfs)"
 236         if [ -z "$pf_list" ] ; then
 237                 return 1;
 238         fi
 239
 240         for dev in $pf_list ; do
 241                 echo 0 > "/sys/bus/pci/devices/$dev/sriov_numvfs"
 242         done
 243
 244         return 0;
 245 }