tools/testing/selftests/cgroup/test_cpuset_prs.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # Test for cpuset v2 partition root state (PRS)
   5 #
   6 # The sched verbose flag can be optionally set so that the console log
   7 # can be examined for the correct setting of scheduling domain.
   8 #
   9
  10 skip_test() {
  11         echo "$1"
  12         echo "Test SKIPPED"
  13         exit 4 # ksft_skip
  14 }
  15
  16 [[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!"
  17
  18
  19 # Get wait_inotify location
  20 WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify
  21
  22 # Find cgroup v2 mount point
  23 CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
  24 [[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!"
  25 SUBPARTS_CPUS=$CGROUP2/.__DEBUG__.cpuset.cpus.subpartitions
  26 CPULIST=$(cat $CGROUP2/cpuset.cpus.effective)
  27
  28 NR_CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//")
  29 [[ $NR_CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!"
  30
  31 # Check to see if /dev/console exists and is writable
  32 if [[ -c /dev/console && -w /dev/console ]]
  33 then
  34         CONSOLE=/dev/console
  35 else
  36         CONSOLE=/dev/null
  37 fi
  38
  39 # Set verbose flag and delay factor
  40 PROG=$1
  41 VERBOSE=0
  42 DELAY_FACTOR=1
  43 SCHED_DEBUG=
  44 while [[ "$1" = -* ]]
  45 do
  46         case "$1" in
  47                 -v) ((VERBOSE++))
  48                     # Enable sched/verbose can slow thing down
  49                     [[ $DELAY_FACTOR -eq 1 ]] &&
  50                         DELAY_FACTOR=2
  51                     ;;
  52                 -d) DELAY_FACTOR=$2
  53                     shift
  54                     ;;
  55                 *)  echo "Usage: $PROG [-v] [-d <delay-factor>"
  56                     exit
  57                     ;;
  58         esac
  59         shift
  60 done
  61
  62 # Set sched verbose flag if available when "-v" option is specified
  63 if [[ $VERBOSE -gt 0 && -d /sys/kernel/debug/sched ]]
  64 then
  65         # Used to restore the original setting during cleanup
  66         SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose)
  67         echo Y > /sys/kernel/debug/sched/verbose
  68 fi
  69
  70 cd $CGROUP2
  71 echo +cpuset > cgroup.subtree_control
  72
  73 #
  74 # If cpuset has been set up and used in child cgroups, we may not be able to
  75 # create partition under root cgroup because of the CPU exclusivity rule.
  76 # So we are going to skip the test if this is the case.
  77 #
  78 [[ -d test ]] || mkdir test
  79 echo 0-6 > test/cpuset.cpus
  80 echo root > test/cpuset.cpus.partition
  81 cat test/cpuset.cpus.partition | grep -q invalid
  82 RESULT=$?
  83 echo member > test/cpuset.cpus.partition
  84 echo "" > test/cpuset.cpus
  85 [[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!"
  86
  87 #
  88 # If isolated CPUs have been reserved at boot time (as shown in
  89 # cpuset.cpus.isolated), these isolated CPUs should be outside of CPUs 0-7
  90 # that will be used by this script for testing purpose. If not, some of
  91 # the tests may fail incorrectly. These isolated CPUs will also be removed
  92 # before being compared with the expected results.
  93 #
  94 BOOT_ISOLCPUS=$(cat $CGROUP2/cpuset.cpus.isolated)
  95 if [[ -n "$BOOT_ISOLCPUS" ]]
  96 then
  97         [[ $(echo $BOOT_ISOLCPUS | sed -e "s/[,-].*//") -le 7 ]] &&
  98                 skip_test "Pre-isolated CPUs ($BOOT_ISOLCPUS) overlap CPUs to be tested"
  99         echo "Pre-isolated CPUs: $BOOT_ISOLCPUS"
 100 fi
 101 cleanup()
 102 {
 103         online_cpus
 104         cd $CGROUP2
 105         rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
 106         rmdir test > /dev/null 2>&1
 107         [[ -n "$SCHED_DEBUG" ]] &&
 108                 echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose
 109 }
 110
 111 # Pause in ms
 112 pause()
 113 {
 114         DELAY=$1
 115         LOOP=0
 116         while [[ $LOOP -lt $DELAY_FACTOR ]]
 117         do
 118                 sleep $DELAY
 119                 ((LOOP++))
 120         done
 121         return 0
 122 }
 123
 124 console_msg()
 125 {
 126         MSG=$1
 127         echo "$MSG"
 128         echo "" > $CONSOLE
 129         echo "$MSG" > $CONSOLE
 130         pause 0.01
 131 }
 132
 133 test_partition()
 134 {
 135         EXPECTED_VAL=$1
 136         echo $EXPECTED_VAL > cpuset.cpus.partition
 137         [[ $? -eq 0 ]] || exit 1
 138         ACTUAL_VAL=$(cat cpuset.cpus.partition)
 139         [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && {
 140                 echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $ACTUAL_VAL"
 141                 echo "Test FAILED"
 142                 exit 1
 143         }
 144 }
 145
 146 test_effective_cpus()
 147 {
 148         EXPECTED_VAL=$1
 149         ACTUAL_VAL=$(cat cpuset.cpus.effective)
 150         [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && {
 151                 echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$ACTUAL_VAL'"
 152                 echo "Test FAILED"
 153                 exit 1
 154         }
 155 }
 156
 157 # Adding current process to cgroup.procs as a test
 158 test_add_proc()
 159 {
 160         OUTSTR="$1"
 161         ERRMSG=$((echo $$ > cgroup.procs) |& cat)
 162         echo $ERRMSG | grep -q "$OUTSTR"
 163         [[ $? -ne 0 ]] && {
 164                 echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'"
 165                 echo "Test FAILED"
 166                 exit 1
 167         }
 168         echo $$ > $CGROUP2/cgroup.procs # Move out the task
 169 }
 170
 171 #
 172 # Cpuset controller state transition test matrix.
 173 #
 174 # Cgroup test hierarchy
 175 #
 176 # root -- A1 -- A2 -- A3
 177 #      +- B1
 178 #
 179 #  P<v> = set cpus.partition (0:member, 1:root, 2:isolated)
 180 #  C<l> = add cpu-list to cpuset.cpus
 181 #  X<l> = add cpu-list to cpuset.cpus.exclusive
 182 #  S<p> = use prefix in subtree_control
 183 #  T    = put a task into cgroup
 184 #  O<c>=<v> = Write <v> to CPU online file of <c>
 185 #
 186 # ECPUs    - effective CPUs of cpusets
 187 # Pstate   - partition root state
 188 # ISOLCPUS - isolated CPUs (<icpus>[,<icpus2>])
 189 #
 190 # Note that if there are 2 fields in ISOLCPUS, the first one is for
 191 # sched-debug matching which includes offline CPUs and single-CPU partitions
 192 # while the second one is for matching cpuset.cpus.isolated.
 193 #
 194 SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1"
 195 TEST_MATRIX=(
 196         #  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
 197         #  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
 198         "   C0-1     .      .    C2-3    S+    C4-5     .      .     0 A2:0-1"
 199         "   C0-1     .      .    C2-3    P1      .      .      .     0 "
 200         "   C0-1     .      .    C2-3   P1:S+ C0-1:P1   .      .     0 "
 201         "   C0-1     .      .    C2-3   P1:S+  C1:P1    .      .     0 "
 202         "  C0-1:S+   .      .    C2-3     .      .      .     P1     0 "
 203         "  C0-1:P1   .      .    C2-3    S+     C1      .      .     0 "
 204         "  C0-1:P1   .      .    C2-3    S+    C1:P1    .      .     0 "
 205         "  C0-1:P1   .      .    C2-3    S+    C1:P1    .     P1     0 "
 206         "  C0-1:P1   .      .    C2-3   C4-5     .      .      .     0 A1:4-5"
 207         "  C0-1:P1   .      .    C2-3  S+:C4-5   .      .      .     0 A1:4-5"
 208         "   C0-1     .      .   C2-3:P1   .      .      .     C2     0 "
 209         "   C0-1     .      .   C2-3:P1   .      .      .    C4-5    0 B1:4-5"
 210         "C0-3:P1:S+ C2-3:P1 .      .      .      .      .      .     0 A1:0-1,A2:2-3"
 211         "C0-3:P1:S+ C2-3:P1 .      .     C1-3    .      .      .     0 A1:1,A2:2-3"
 212         "C2-3:P1:S+  C3:P1  .      .     C3      .      .      .     0 A1:,A2:3 A1:P1,A2:P1"
 213         "C2-3:P1:S+  C3:P1  .      .     C3      P0     .      .     0 A1:3,A2:3 A1:P1,A2:P0"
 214         "C2-3:P1:S+  C2:P1  .      .     C2-4    .      .      .     0 A1:3-4,A2:2"
 215         "C2-3:P1:S+  C3:P1  .      .     C3      .      .     C0-2   0 A1:,B1:0-2 A1:P1,A2:P1"
 216         "$SETUP_A123_PARTITIONS    .     C2-3    .      .      .     0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 217
 218         # CPU offlining cases:
 219         "   C0-1     .      .    C2-3    S+    C4-5     .     O2=0   0 A1:0-1,B1:3"
 220         "C0-3:P1:S+ C2-3:P1 .      .     O2=0    .      .      .     0 A1:0-1,A2:3"
 221         "C0-3:P1:S+ C2-3:P1 .      .     O2=0   O2=1    .      .     0 A1:0-1,A2:2-3"
 222         "C0-3:P1:S+ C2-3:P1 .      .     O1=0    .      .      .     0 A1:0,A2:2-3"
 223         "C0-3:P1:S+ C2-3:P1 .      .     O1=0   O1=1    .      .     0 A1:0-1,A2:2-3"
 224         "C2-3:P1:S+  C3:P1  .      .     O3=0   O3=1    .      .     0 A1:2,A2:3 A1:P1,A2:P1"
 225         "C2-3:P1:S+  C3:P2  .      .     O3=0   O3=1    .      .     0 A1:2,A2:3 A1:P1,A2:P2"
 226         "C2-3:P1:S+  C3:P1  .      .     O2=0   O2=1    .      .     0 A1:2,A2:3 A1:P1,A2:P1"
 227         "C2-3:P1:S+  C3:P2  .      .     O2=0   O2=1    .      .     0 A1:2,A2:3 A1:P1,A2:P2"
 228         "C2-3:P1:S+  C3:P1  .      .     O2=0    .      .      .     0 A1:,A2:3 A1:P1,A2:P1"
 229         "C2-3:P1:S+  C3:P1  .      .     O3=0    .      .      .     0 A1:2,A2: A1:P1,A2:P1"
 230         "C2-3:P1:S+  C3:P1  .      .    T:O2=0   .      .      .     0 A1:3,A2:3 A1:P1,A2:P-1"
 231         "C2-3:P1:S+  C3:P1  .      .      .    T:O3=0   .      .     0 A1:2,A2:2 A1:P1,A2:P-1"
 232         "$SETUP_A123_PARTITIONS    .     O1=0    .      .      .     0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 233         "$SETUP_A123_PARTITIONS    .     O2=0    .      .      .     0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
 234         "$SETUP_A123_PARTITIONS    .     O3=0    .      .      .     0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1"
 235         "$SETUP_A123_PARTITIONS    .    T:O1=0   .      .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
 236         "$SETUP_A123_PARTITIONS    .      .    T:O2=0   .      .     0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1"
 237         "$SETUP_A123_PARTITIONS    .      .      .    T:O3=0   .     0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1"
 238         "$SETUP_A123_PARTITIONS    .    T:O1=0  O1=1    .      .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 239         "$SETUP_A123_PARTITIONS    .      .    T:O2=0  O2=1    .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 240         "$SETUP_A123_PARTITIONS    .      .      .    T:O3=0  O3=1   0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 241         "$SETUP_A123_PARTITIONS    .    T:O1=0  O2=0   O1=1    .     0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
 242         "$SETUP_A123_PARTITIONS    .    T:O1=0  O2=0   O2=1    .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
 243
 244         #  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
 245         #  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
 246         #
 247         # Remote partition and cpuset.cpus.exclusive tests
 248         #
 249         " C0-3:S+ C1-3:S+ C2-3     .    X2-3     .      .      .     0 A1:0-3,A2:1-3,A3:2-3,XA1:2-3"
 250         " C0-3:S+ C1-3:S+ C2-3     .    X2-3  X2-3:P2   .      .     0 A1:0-1,A2:2-3,A3:2-3 A1:P0,A2:P2 2-3"
 251         " C0-3:S+ C1-3:S+ C2-3     .    X2-3   X3:P2    .      .     0 A1:0-2,A2:3,A3:3 A1:P0,A2:P2 3"
 252         " C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3  X2-3:P2   .     0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
 253         " C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3 X2-3:P2:C3 .     0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
 254         " C0-3:S+ C1-3:S+ C2-3   C2-3     .      .      .      P2    0 A1:0-3,A2:1-3,A3:2-3,B1:2-3 A1:P0,A3:P0,B1:P-2"
 255         " C0-3:S+ C1-3:S+ C2-3   C4-5     .      .      .      P2    0 B1:4-5 B1:P2 4-5"
 256         " C0-3:S+ C1-3:S+ C2-3    C4    X2-3   X2-3  X2-3:P2   P2    0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"
 257         " C0-3:S+ C1-3:S+ C2-3    C4    X2-3   X2-3 X2-3:P2:C1-3 P2  0 A3:2-3,B1:4 A3:P2,B1:P2 2-4"
 258         " C0-3:S+ C1-3:S+ C2-3    C4    X1-3  X1-3:P2   P2     .     0 A2:1,A3:2-3 A2:P2,A3:P2 1-3"
 259         " C0-3:S+ C1-3:S+ C2-3    C4    X2-3   X2-3  X2-3:P2 P2:C4-5 0 A3:2-3,B1:4-5 A3:P2,B1:P2 2-5"
 260         " C4:X0-3:S+ X1-3:S+ X2-3  .      .      P2     .      .     0 A1:4,A2:1-3,A3:1-3 A2:P2 1-3"
 261         " C4:X0-3:S+ X1-3:S+ X2-3  .      .      .      P2     .     0 A1:4,A2:4,A3:2-3 A3:P2 2-3"
 262
 263         # Nested remote/local partition tests
 264         " C0-3:S+ C1-3:S+ C2-3   C4-5   X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:,A3:2-3,B1:4-5 \
 265                                                                        A1:P0,A2:P1,A3:P2,B1:P1 2-3"
 266         " C0-3:S+ C1-3:S+ C2-3    C4    X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:,A3:2-3,B1:4 \
 267                                                                        A1:P0,A2:P1,A3:P2,B1:P1 2-4,2-3"
 268         " C0-3:S+ C1-3:S+ C2-3    C4    X2-3  X2-3:P1    .     P1    0 A1:0-1,A2:2-3,A3:2-3,B1:4 \
 269                                                                        A1:P0,A2:P1,A3:P0,B1:P1"
 270         " C0-3:S+ C1-3:S+  C3     C4    X2-3  X2-3:P1   P2     P1    0 A1:0-1,A2:2,A3:3,B1:4 \
 271                                                                        A1:P0,A2:P1,A3:P2,B1:P1 2-4,3"
 272         " C0-4:S+ C1-4:S+ C2-4     .    X2-4  X2-4:P2  X4:P1    .    0 A1:0-1,A2:2-3,A3:4 \
 273                                                                        A1:P0,A2:P2,A3:P1 2-4,2-3"
 274         " C0-4:S+ C1-4:S+ C2-4     .    X2-4  X2-4:P2 X3-4:P1   .    0 A1:0-1,A2:2,A3:3-4 \
 275                                                                        A1:P0,A2:P2,A3:P1 2"
 276         " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
 277                                    .      .      X5      .      .    0 A1:0-4,A2:1-4,A3:2-4 \
 278                                                                        A1:P0,A2:P-2,A3:P-1"
 279         " C0-4:X2-4:S+ C1-4:X2-4:S+:P2 C2-4:X4:P1 \
 280                                    .      .      .      X1      .    0 A1:0-1,A2:2-4,A3:2-4 \
 281                                                                        A1:P0,A2:P2,A3:P-1 2-4"
 282
 283         # Remote partition offline tests
 284         " C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3 X2-3:P2:O2=0 .   0 A1:0-1,A2:1,A3:3 A1:P0,A3:P2 2-3"
 285         " C0-3:S+ C1-3:S+ C2-3     .    X2-3   X2-3 X2-3:P2:O2=0 O2=1 0 A1:0-1,A2:1,A3:2-3 A1:P0,A3:P2 2-3"
 286         " C0-3:S+ C1-3:S+  C3      .    X2-3   X2-3    P2:O3=0   .   0 A1:0-2,A2:1-2,A3: A1:P0,A3:P2 3"
 287         " C0-3:S+ C1-3:S+  C3      .    X2-3   X2-3   T:P2:O3=0  .   0 A1:0-2,A2:1-2,A3:1-2 A1:P0,A3:P-2 3,"
 288
 289         # An invalidated remote partition cannot self-recover from hotplug
 290         " C0-3:S+ C1-3:S+  C2      .    X2-3   X2-3   T:P2:O2=0 O2=1 0 A1:0-3,A2:1-3,A3:2 A1:P0,A3:P-2"
 291
 292         # cpus.exclusive.effective clearing test
 293         " C0-3:S+ C1-3:S+  C2      .   X2-3:X    .      .      .     0 A1:0-3,A2:1-3,A3:2,XA1:"
 294
 295         # Invalid to valid remote partition transition test
 296         " C0-3:S+   C1-3    .      .      .    X3:P2    .      .     0 A1:0-3,A2:1-3,XA2: A2:P-2"
 297         " C0-3:S+ C1-3:X3:P2
 298                             .      .    X2-3    P2      .      .     0 A1:0-2,A2:3,XA2:3 A2:P2 3"
 299
 300         # Invalid to valid local partition direct transition tests
 301         " C1-3:S+:P2 X4:P2  .      .      .      .      .      .     0 A1:1-3,XA1:1-3,A2:1-3:XA2: A1:P2,A2:P-2 1-3"
 302         " C1-3:S+:P2 X4:P2  .      .      .    X3:P2    .      .     0 A1:1-2,XA1:1-3,A2:3:XA2:3 A1:P2,A2:P2 1-3"
 303         "  C0-3:P2   .      .    C4-6   C0-4     .      .      .     0 A1:0-4,B1:4-6 A1:P-2,B1:P0"
 304         "  C0-3:P2   .      .    C4-6 C0-4:C0-3  .      .      .     0 A1:0-3,B1:4-6 A1:P2,B1:P0 0-3"
 305         "  C0-3:P2   .      .  C3-5:C4-5  .      .      .      .     0 A1:0-3,B1:4-5 A1:P2,B1:P0 0-3"
 306
 307         # Local partition invalidation tests
 308         " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
 309                                    .      .      .      .      .     0 A1:1,A2:2,A3:3 A1:P2,A2:P2,A3:P2 1-3"
 310         " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
 311                                    .      .     X4      .      .     0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"
 312         " C0-3:X1-3:S+:P2 C1-3:X2-3:S+:P2 C2-3:X3:P2 \
 313                                    .      .    C4:X     .      .     0 A1:1-3,A2:1-3,A3:2-3,XA2:,XA3: A1:P2,A2:P-2,A3:P-2 1-3"
 314         # Local partition CPU change tests
 315         " C0-5:S+:P2 C4-5:S+:P1 .  .      .    C3-5     .      .     0 A1:0-2,A2:3-5 A1:P2,A2:P1 0-2"
 316         " C0-5:S+:P2 C4-5:S+:P1 .  .    C1-5     .      .      .     0 A1:1-3,A2:4-5 A1:P2,A2:P1 1-3"
 317
 318         # cpus_allowed/exclusive_cpus update tests
 319         " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 320                                    .    X:C4     .      P2     .     0 A1:4,A2:4,XA2:,XA3:,A3:4 \
 321                                                                        A1:P0,A3:P-2"
 322         " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 323                                    .     X1      .      P2     .     0 A1:0-3,A2:1-3,XA1:1,XA2:,XA3:,A3:2-3 \
 324                                                                        A1:P0,A3:P-2"
 325         " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3 \
 326                                    .      .     X3      P2     .     0 A1:0-2,A2:1-2,XA2:3,XA3:3,A3:3 \
 327                                                                        A1:P0,A3:P2 3"
 328         " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
 329                                    .      .     X3      .      .     0 A1:0-3,A2:1-3,XA2:3,XA3:3,A3:2-3 \
 330                                                                        A1:P0,A3:P-2"
 331         " C0-3:X2-3:S+ C1-3:X2-3:S+ C2-3:X2-3:P2 \
 332                                    .     X4      .      .      .     0 A1:0-3,A2:1-3,A3:2-3,XA1:4,XA2:,XA3 \
 333                                                                        A1:P0,A3:P-2"
 334
 335         #  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
 336         #  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
 337         #
 338         # Incorrect change to cpuset.cpus[.exclusive] invalidates partition root
 339         #
 340         # Adding CPUs to partition root that are not in parent's
 341         # cpuset.cpus is allowed, but those extra CPUs are ignored.
 342         "C2-3:P1:S+ C3:P1   .      .      .     C2-4    .      .     0 A1:,A2:2-3 A1:P1,A2:P1"
 343
 344         # Taking away all CPUs from parent or itself if there are tasks
 345         # will make the partition invalid.
 346         "C2-3:P1:S+  C3:P1  .      .      T     C2-3    .      .     0 A1:2-3,A2:2-3 A1:P1,A2:P-1"
 347         " C3:P1:S+    C3    .      .      T      P1     .      .     0 A1:3,A2:3 A1:P1,A2:P-1"
 348         "$SETUP_A123_PARTITIONS    .    T:C2-3   .      .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
 349         "$SETUP_A123_PARTITIONS    . T:C2-3:C1-3 .      .      .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
 350
 351         # Changing a partition root to member makes child partitions invalid
 352         "C2-3:P1:S+  C3:P1  .      .      P0     .      .      .     0 A1:2-3,A2:3 A1:P0,A2:P-1"
 353         "$SETUP_A123_PARTITIONS    .     C2-3    P0     .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1"
 354
 355         # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long
 356         # as they overlap.
 357         "C2-3:P1:S+  .      .      .      .   C3-4:P1   .      .     0 A1:2,A2:3 A1:P1,A2:P1"
 358
 359         # Deletion of CPUs distributed to child cgroup is allowed.
 360         "C0-1:P1:S+ C1      .    C2-3   C4-5     .      .      .     0 A1:4-5,A2:4-5"
 361
 362         # To become a valid partition root, cpuset.cpus must overlap parent's
 363         # cpuset.cpus.
 364         "  C0-1:P1   .      .    C2-3    S+   C4-5:P1   .      .     0 A1:0-1,A2:0-1 A1:P1,A2:P-1"
 365
 366         # Enabling partition with child cpusets is allowed
 367         "  C0-1:S+  C1      .    C2-3    P1      .      .      .     0 A1:0-1,A2:1 A1:P1"
 368
 369         # A partition root with non-partition root parent is invalid, but it
 370         # can be made valid if its parent becomes a partition root too.
 371         "  C0-1:S+  C1      .    C2-3     .      P2     .      .     0 A1:0-1,A2:1 A1:P0,A2:P-2"
 372         "  C0-1:S+ C1:P2    .    C2-3     P1     .      .      .     0 A1:0,A2:1 A1:P1,A2:P2"
 373
 374         # A non-exclusive cpuset.cpus change will invalidate partition and its siblings
 375         "  C0-1:P1   .      .    C2-3   C0-2     .      .      .     0 A1:0-2,B1:2-3 A1:P-1,B1:P0"
 376         "  C0-1:P1   .      .  P1:C2-3  C0-2     .      .      .     0 A1:0-2,B1:2-3 A1:P-1,B1:P-1"
 377         "   C0-1     .      .  P1:C2-3  C0-2     .      .      .     0 A1:0-2,B1:2-3 A1:P0,B1:P-1"
 378
 379         # cpuset.cpus can overlap with sibling cpuset.cpus.exclusive but not subsumed by it
 380         "   C0-3     .      .    C4-5     X5     .      .      .     0 A1:0-3,B1:4-5"
 381
 382         # Child partition root that try to take all CPUs from parent partition
 383         # with tasks will remain invalid.
 384         " C1-4:P1:S+ P1     .      .       .     .      .      .     0 A1:1-4,A2:1-4 A1:P1,A2:P-1"
 385         " C1-4:P1:S+ P1     .      .       .   C1-4     .      .     0 A1,A2:1-4 A1:P1,A2:P1"
 386         " C1-4:P1:S+ P1     .      .       T   C1-4     .      .     0 A1:1-4,A2:1-4 A1:P1,A2:P-1"
 387
 388         # Clearing of cpuset.cpus with a preset cpuset.cpus.exclusive shouldn't
 389         # affect cpuset.cpus.exclusive.effective.
 390         " C1-4:X3:S+ C1:X3  .      .       .     C      .      .     0 A2:1-4,XA2:3"
 391
 392         #  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS
 393         #  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ --------
 394         # Failure cases:
 395
 396         # A task cannot be added to a partition with no cpu
 397         "C2-3:P1:S+  C3:P1  .      .    O2=0:T   .      .      .     1 A1:,A2:3 A1:P1,A2:P1"
 398
 399         # Changes to cpuset.cpus.exclusive that violate exclusivity rule is rejected
 400         "   C0-3     .      .    C4-5   X0-3     .      .     X3-5   1 A1:0-3,B1:4-5"
 401
 402         # cpuset.cpus cannot be a subset of sibling cpuset.cpus.exclusive
 403         "   C0-3     .      .    C4-5   X3-5     .      .      .     1 A1:0-3,B1:4-5"
 404 )
 405
 406 #
 407 # Write to the cpu online file
 408 #  $1 - <c>=<v> where <c> = cpu number, <v> value to be written
 409 #
 410 write_cpu_online()
 411 {
 412         CPU=${1%=*}
 413         VAL=${1#*=}
 414         CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online
 415         if [[ $VAL -eq 0 ]]
 416         then
 417                 OFFLINE_CPUS="$OFFLINE_CPUS $CPU"
 418         else
 419                 [[ -n "$OFFLINE_CPUS" ]] && {
 420                         OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\
 421                                         sort | uniq -u)
 422                 }
 423         fi
 424         echo $VAL > $CPUFILE
 425         pause 0.05
 426 }
 427
 428 #
 429 # Set controller state
 430 #  $1 - cgroup directory
 431 #  $2 - state
 432 #  $3 - showerr
 433 #
 434 # The presence of ":" in state means transition from one to the next.
 435 #
 436 set_ctrl_state()
 437 {
 438         TMPMSG=/tmp/.msg_$$
 439         CGRP=$1
 440         STATE=$2
 441         SHOWERR=${3}
 442         CTRL=${CTRL:=$CONTROLLER}
 443         HASERR=0
 444         REDIRECT="2> $TMPMSG"
 445         [[ -z "$STATE" || "$STATE" = '.' ]] && return 0
 446         [[ $VERBOSE -gt 0 ]] && SHOWERR=1
 447
 448         rm -f $TMPMSG
 449         for CMD in $(echo $STATE | sed -e "s/:/ /g")
 450         do
 451                 TFILE=$CGRP/cgroup.procs
 452                 SFILE=$CGRP/cgroup.subtree_control
 453                 PFILE=$CGRP/cpuset.cpus.partition
 454                 CFILE=$CGRP/cpuset.cpus
 455                 XFILE=$CGRP/cpuset.cpus.exclusive
 456                 S=$(expr substr $CMD 1 1)
 457                 if [[ $S = S ]]
 458                 then
 459                         PREFIX=${CMD#?}
 460                         COMM="echo ${PREFIX}${CTRL} > $SFILE"
 461                         eval $COMM $REDIRECT
 462                 elif [[ $S = X ]]
 463                 then
 464                         CPUS=${CMD#?}
 465                         COMM="echo $CPUS > $XFILE"
 466                         eval $COMM $REDIRECT
 467                 elif [[ $S = C ]]
 468                 then
 469                         CPUS=${CMD#?}
 470                         COMM="echo $CPUS > $CFILE"
 471                         eval $COMM $REDIRECT
 472                 elif [[ $S = P ]]
 473                 then
 474                         VAL=${CMD#?}
 475                         case $VAL in
 476                         0)  VAL=member
 477                             ;;
 478                         1)  VAL=root
 479                             ;;
 480                         2)  VAL=isolated
 481                             ;;
 482                         *)
 483                             echo "Invalid partition state - $VAL"
 484                             exit 1
 485                             ;;
 486                         esac
 487                         COMM="echo $VAL > $PFILE"
 488                         eval $COMM $REDIRECT
 489                 elif [[ $S = O ]]
 490                 then
 491                         VAL=${CMD#?}
 492                         write_cpu_online $VAL
 493                 elif [[ $S = T ]]
 494                 then
 495                         COMM="echo 0 > $TFILE"
 496                         eval $COMM $REDIRECT
 497                 fi
 498                 RET=$?
 499                 [[ $RET -ne 0 ]] && {
 500                         [[ -n "$SHOWERR" ]] && {
 501                                 echo "$COMM"
 502                                 cat $TMPMSG
 503                         }
 504                         HASERR=1
 505                 }
 506                 pause 0.01
 507                 rm -f $TMPMSG
 508         done
 509         return $HASERR
 510 }
 511
 512 set_ctrl_state_noerr()
 513 {
 514         CGRP=$1
 515         STATE=$2
 516         [[ -d $CGRP ]] || mkdir $CGRP
 517         set_ctrl_state $CGRP $STATE 1
 518         [[ $? -ne 0 ]] && {
 519                 echo "ERROR: Failed to set $2 to cgroup $1!"
 520                 exit 1
 521         }
 522 }
 523
 524 online_cpus()
 525 {
 526         [[ -n "OFFLINE_CPUS" ]] && {
 527                 for C in $OFFLINE_CPUS
 528                 do
 529                         write_cpu_online ${C}=1
 530                 done
 531         }
 532 }
 533
 534 #
 535 # Return 1 if the list of effective cpus isn't the same as the initial list.
 536 #
 537 reset_cgroup_states()
 538 {
 539         echo 0 > $CGROUP2/cgroup.procs
 540         online_cpus
 541         rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
 542         pause 0.02
 543         set_ctrl_state . R-
 544         pause 0.01
 545 }
 546
 547 dump_states()
 548 {
 549         for DIR in . A1 A1/A2 A1/A2/A3 B1
 550         do
 551                 CPUS=$DIR/cpuset.cpus
 552                 ECPUS=$DIR/cpuset.cpus.effective
 553                 XCPUS=$DIR/cpuset.cpus.exclusive
 554                 XECPUS=$DIR/cpuset.cpus.exclusive.effective
 555                 PRS=$DIR/cpuset.cpus.partition
 556                 PCPUS=$DIR/.__DEBUG__.cpuset.cpus.subpartitions
 557                 ISCPUS=$DIR/cpuset.cpus.isolated
 558                 [[ -e $CPUS   ]] && echo "$CPUS: $(cat $CPUS)"
 559                 [[ -e $XCPUS  ]] && echo "$XCPUS: $(cat $XCPUS)"
 560                 [[ -e $ECPUS  ]] && echo "$ECPUS: $(cat $ECPUS)"
 561                 [[ -e $XECPUS ]] && echo "$XECPUS: $(cat $XECPUS)"
 562                 [[ -e $PRS    ]] && echo "$PRS: $(cat $PRS)"
 563                 [[ -e $PCPUS  ]] && echo "$PCPUS: $(cat $PCPUS)"
 564                 [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)"
 565         done
 566 }
 567
 568 #
 569 # Check effective cpus
 570 # $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]*
 571 #
 572 check_effective_cpus()
 573 {
 574         CHK_STR=$1
 575         for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
 576         do
 577                 set -- $(echo $CHK | sed -e "s/:/ /g")
 578                 CGRP=$1
 579                 CPUS=$2
 580                 if [[ $CGRP = X* ]]
 581                 then
 582                         CGRP=${CGRP#X}
 583                         FILE=cpuset.cpus.exclusive.effective
 584                 else
 585                         FILE=cpuset.cpus.effective
 586                 fi
 587                 [[ $CGRP = A2 ]] && CGRP=A1/A2
 588                 [[ $CGRP = A3 ]] && CGRP=A1/A2/A3
 589                 [[ -e $CGRP/$FILE ]] || return 1
 590                 [[ $CPUS = $(cat $CGRP/$FILE) ]] || return 1
 591         done
 592 }
 593
 594 #
 595 # Check cgroup states
 596 #  $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]*
 597 #
 598 check_cgroup_states()
 599 {
 600         CHK_STR=$1
 601         for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
 602         do
 603                 set -- $(echo $CHK | sed -e "s/:/ /g")
 604                 CGRP=$1
 605                 CGRP_DIR=$CGRP
 606                 STATE=$2
 607                 FILE=
 608                 EVAL=$(expr substr $STATE 2 2)
 609                 [[ $CGRP = A2 ]] && CGRP_DIR=A1/A2
 610                 [[ $CGRP = A3 ]] && CGRP_DIR=A1/A2/A3
 611
 612                 case $STATE in
 613                         P*) FILE=$CGRP_DIR/cpuset.cpus.partition
 614                             ;;
 615                         *)  echo "Unknown state: $STATE!"
 616                             exit 1
 617                             ;;
 618                 esac
 619                 VAL=$(cat $FILE)
 620
 621                 case "$VAL" in
 622                         member) VAL=0
 623                                 ;;
 624                         root)   VAL=1
 625                                 ;;
 626                         isolated)
 627                                 VAL=2
 628                                 ;;
 629                         "root invalid"*)
 630                                 VAL=-1
 631                                 ;;
 632                         "isolated invalid"*)
 633                                 VAL=-2
 634                                 ;;
 635                 esac
 636                 [[ $EVAL != $VAL ]] && return 1
 637
 638                 #
 639                 # For root partition, dump sched-domains info to console if
 640                 # verbose mode set for manual comparison with sched debug info.
 641                 #
 642                 [[ $VAL -eq 1 && $VERBOSE -gt 0 ]] && {
 643                         DOMS=$(cat $CGRP_DIR/cpuset.cpus.effective)
 644                         [[ -n "$DOMS" ]] &&
 645                                 echo " [$CGRP] sched-domain: $DOMS" > $CONSOLE
 646                 }
 647         done
 648         return 0
 649 }
 650
 651 #
 652 # Get isolated (including offline) CPUs by looking at
 653 # /sys/kernel/debug/sched/domains and cpuset.cpus.isolated control file,
 654 # if available, and compare that with the expected value.
 655 #
 656 # Note that isolated CPUs from the sched/domains context include offline
 657 # CPUs as well as CPUs in non-isolated 1-CPU partition. Those CPUs may
 658 # not be included in the cpuset.cpus.isolated control file which contains
 659 # only CPUs in isolated partitions as well as those that are isolated at
 660 # boot time.
 661 #
 662 # $1 - expected isolated cpu list(s) <isolcpus1>{,<isolcpus2>}
 663 # <isolcpus1> - expected sched/domains value
 664 # <isolcpus2> - cpuset.cpus.isolated value = <isolcpus1> if not defined
 665 #
 666 check_isolcpus()
 667 {
 668         EXPECT_VAL=$1
 669         ISOLCPUS=
 670         LASTISOLCPU=
 671         SCHED_DOMAINS=/sys/kernel/debug/sched/domains
 672         ISCPUS=${CGROUP2}/cpuset.cpus.isolated
 673         if [[ $EXPECT_VAL = . ]]
 674         then
 675                 EXPECT_VAL=
 676                 EXPECT_VAL2=
 677         elif [[ $(expr $EXPECT_VAL : ".*,.*") > 0 ]]
 678         then
 679                 set -- $(echo $EXPECT_VAL | sed -e "s/,/ /g")
 680                 EXPECT_VAL=$1
 681                 EXPECT_VAL2=$2
 682         else
 683                 EXPECT_VAL2=$EXPECT_VAL
 684         fi
 685
 686         #
 687         # Check cpuset.cpus.isolated cpumask
 688         #
 689         if [[ -z "$BOOT_ISOLCPUS" ]]
 690         then
 691                 ISOLCPUS=$(cat $ISCPUS)
 692         else
 693                 ISOLCPUS=$(cat $ISCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")
 694         fi
 695         [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && {
 696                 # Take a 50ms pause and try again
 697                 pause 0.05
 698                 ISOLCPUS=$(cat $ISCPUS)
 699         }
 700         [[ "$EXPECT_VAL2" != "$ISOLCPUS" ]] && return 1
 701         ISOLCPUS=
 702
 703         #
 704         # Use the sched domain in debugfs to check isolated CPUs, if available
 705         #
 706         [[ -d $SCHED_DOMAINS ]] || return 0
 707
 708         for ((CPU=0; CPU < $NR_CPUS; CPU++))
 709         do
 710                 [[ -n "$(ls ${SCHED_DOMAINS}/cpu$CPU)" ]] && continue
 711
 712                 if [[ -z "$LASTISOLCPU" ]]
 713                 then
 714                         ISOLCPUS=$CPU
 715                         LASTISOLCPU=$CPU
 716                 elif [[ "$LASTISOLCPU" -eq $((CPU - 1)) ]]
 717                 then
 718                         echo $ISOLCPUS | grep -q "\<$LASTISOLCPU\$"
 719                         if [[ $? -eq 0 ]]
 720                         then
 721                                 ISOLCPUS=${ISOLCPUS}-
 722                         fi
 723                         LASTISOLCPU=$CPU
 724                 else
 725                         if [[ $ISOLCPUS = *- ]]
 726                         then
 727                                 ISOLCPUS=${ISOLCPUS}$LASTISOLCPU
 728                         fi
 729                         ISOLCPUS=${ISOLCPUS},$CPU
 730                         LASTISOLCPU=$CPU
 731                 fi
 732         done
 733         [[ "$ISOLCPUS" = *- ]] && ISOLCPUS=${ISOLCPUS}$LASTISOLCPU
 734         [[ -n "BOOT_ISOLCPUS" ]] &&
 735                 ISOLCPUS=$(echo $ISOLCPUS | sed -e "s/,*$BOOT_ISOLCPUS//")
 736
 737         [[ "$EXPECT_VAL" = "$ISOLCPUS" ]]
 738 }
 739
 740 test_fail()
 741 {
 742         TESTNUM=$1
 743         TESTTYPE=$2
 744         ADDINFO=$3
 745         echo "Test $TEST[$TESTNUM] failed $TESTTYPE check!"
 746         [[ -n "$ADDINFO" ]] && echo "*** $ADDINFO ***"
 747         eval echo \${$TEST[$I]}
 748         echo
 749         dump_states
 750         exit 1
 751 }
 752
 753 #
 754 # Check to see if there are unexpected isolated CPUs left beyond the boot
 755 # time isolated ones.
 756 #
 757 null_isolcpus_check()
 758 {
 759         [[ $VERBOSE -gt 0 ]] || return 0
 760         # Retry a few times before printing error
 761         RETRY=0
 762         while [[ $RETRY -lt 8 ]]
 763         do
 764                 pause 0.02
 765                 check_isolcpus "."
 766                 [[ $? -eq 0 ]] && return 0
 767                 ((RETRY++))
 768         done
 769         echo "Unexpected isolated CPUs: $ISOLCPUS"
 770         dump_states
 771         exit 1
 772 }
 773
 774 #
 775 # Run cpuset state transition test
 776 #  $1 - test matrix name
 777 #
 778 # This test is somewhat fragile as delays (sleep x) are added in various
 779 # places to make sure state changes are fully propagated before the next
 780 # action. These delays may need to be adjusted if running in a slower machine.
 781 #
 782 run_state_test()
 783 {
 784         TEST=$1
 785         CONTROLLER=cpuset
 786         I=0
 787         eval CNT="\${#$TEST[@]}"
 788
 789         reset_cgroup_states
 790         console_msg "Running state transition test ..."
 791
 792         while [[ $I -lt $CNT ]]
 793         do
 794                 echo "Running test $I ..." > $CONSOLE
 795                 [[ $VERBOSE -gt 1 ]] && {
 796                         echo ""
 797                         eval echo \${$TEST[$I]}
 798                 }
 799                 eval set -- "\${$TEST[$I]}"
 800                 OLD_A1=$1
 801                 OLD_A2=$2
 802                 OLD_A3=$3
 803                 OLD_B1=$4
 804                 NEW_A1=$5
 805                 NEW_A2=$6
 806                 NEW_A3=$7
 807                 NEW_B1=$8
 808                 RESULT=$9
 809                 ECPUS=${10}
 810                 STATES=${11}
 811                 ICPUS=${12}
 812
 813                 set_ctrl_state_noerr B1       $OLD_B1
 814                 set_ctrl_state_noerr A1       $OLD_A1
 815                 set_ctrl_state_noerr A1/A2    $OLD_A2
 816                 set_ctrl_state_noerr A1/A2/A3 $OLD_A3
 817                 RETVAL=0
 818                 set_ctrl_state A1       $NEW_A1; ((RETVAL += $?))
 819                 set_ctrl_state A1/A2    $NEW_A2; ((RETVAL += $?))
 820                 set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?))
 821                 set_ctrl_state B1       $NEW_B1; ((RETVAL += $?))
 822
 823                 [[ $RETVAL -ne $RESULT ]] && test_fail $I result
 824
 825                 [[ -n "$ECPUS" && "$ECPUS" != . ]] && {
 826                         check_effective_cpus $ECPUS
 827                         [[ $? -ne 0 ]] && test_fail $I "effective CPU"
 828                 }
 829
 830                 [[ -n "$STATES" && "$STATES" != . ]] && {
 831                         check_cgroup_states $STATES
 832                         [[ $? -ne 0 ]] && test_fail $I states
 833                 }
 834
 835                 # Compare the expected isolated CPUs with the actual ones,
 836                 # if available
 837                 [[ -n "$ICPUS" ]] && {
 838                         check_isolcpus $ICPUS
 839                         [[ $? -ne 0 ]] && test_fail $I "isolated CPU" \
 840                                 "Expect $ICPUS, get $ISOLCPUS instead"
 841                 }
 842                 reset_cgroup_states
 843                 #
 844                 # Check to see if effective cpu list changes
 845                 #
 846                 NEWLIST=$(cat cpuset.cpus.effective)
 847                 RETRY=0
 848                 while [[ $NEWLIST != $CPULIST && $RETRY -lt 8 ]]
 849                 do
 850                         # Wait a bit longer & recheck a few times
 851                         pause 0.02
 852                         ((RETRY++))
 853                         NEWLIST=$(cat cpuset.cpus.effective)
 854                 done
 855                 [[ $NEWLIST != $CPULIST ]] && {
 856                         echo "Effective cpus changed to $NEWLIST after test $I!"
 857                         exit 1
 858                 }
 859                 null_isolcpus_check
 860                 [[ $VERBOSE -gt 0 ]] && echo "Test $I done."
 861                 ((I++))
 862         done
 863         echo "All $I tests of $TEST PASSED."
 864 }
 865
 866 #
 867 # Testing the new "isolated" partition root type
 868 #
 869 test_isolated()
 870 {
 871         cd $CGROUP2/test
 872         echo 2-3 > cpuset.cpus
 873         TYPE=$(cat cpuset.cpus.partition)
 874         [[ $TYPE = member ]] || echo member > cpuset.cpus.partition
 875
 876         console_msg "Change from member to root"
 877         test_partition root
 878
 879         console_msg "Change from root to isolated"
 880         test_partition isolated
 881
 882         console_msg "Change from isolated to member"
 883         test_partition member
 884
 885         console_msg "Change from member to isolated"
 886         test_partition isolated
 887
 888         console_msg "Change from isolated to root"
 889         test_partition root
 890
 891         console_msg "Change from root to member"
 892         test_partition member
 893
 894         #
 895         # Testing partition root with no cpu
 896         #
 897         console_msg "Distribute all cpus to child partition"
 898         echo +cpuset > cgroup.subtree_control
 899         test_partition root
 900
 901         mkdir A1
 902         cd A1
 903         echo 2-3 > cpuset.cpus
 904         test_partition root
 905         test_effective_cpus 2-3
 906         cd ..
 907         test_effective_cpus ""
 908
 909         console_msg "Moving task to partition test"
 910         test_add_proc "No space left"
 911         cd A1
 912         test_add_proc ""
 913         cd ..
 914
 915         console_msg "Shrink and expand child partition"
 916         cd A1
 917         echo 2 > cpuset.cpus
 918         cd ..
 919         test_effective_cpus 3
 920         cd A1
 921         echo 2-3 > cpuset.cpus
 922         cd ..
 923         test_effective_cpus ""
 924
 925         # Cleaning up
 926         console_msg "Cleaning up"
 927         echo $$ > $CGROUP2/cgroup.procs
 928         [[ -d A1 ]] && rmdir A1
 929         null_isolcpus_check
 930 }
 931
 932 #
 933 # Wait for inotify event for the given file and read it
 934 # $1: cgroup file to wait for
 935 # $2: file to store the read result
 936 #
 937 wait_inotify()
 938 {
 939         CGROUP_FILE=$1
 940         OUTPUT_FILE=$2
 941
 942         $WAIT_INOTIFY $CGROUP_FILE
 943         cat $CGROUP_FILE > $OUTPUT_FILE
 944 }
 945
 946 #
 947 # Test if inotify events are properly generated when going into and out of
 948 # invalid partition state.
 949 #
 950 test_inotify()
 951 {
 952         ERR=0
 953         PRS=/tmp/.prs_$$
 954         cd $CGROUP2/test
 955         [[ -f $WAIT_INOTIFY ]] || {
 956                 echo "wait_inotify not found, inotify test SKIPPED."
 957                 return
 958         }
 959
 960         pause 0.01
 961         echo 1 > cpuset.cpus
 962         echo 0 > cgroup.procs
 963         echo root > cpuset.cpus.partition
 964         pause 0.01
 965         rm -f $PRS
 966         wait_inotify $PWD/cpuset.cpus.partition $PRS &
 967         pause 0.01
 968         set_ctrl_state . "O1=0"
 969         pause 0.01
 970         check_cgroup_states ".:P-1"
 971         if [[ $? -ne 0 ]]
 972         then
 973                 echo "FAILED: Inotify test - partition not invalid"
 974                 ERR=1
 975         elif [[ ! -f $PRS ]]
 976         then
 977                 echo "FAILED: Inotify test - event not generated"
 978                 ERR=1
 979                 kill %1
 980         elif [[ $(cat $PRS) != "root invalid"* ]]
 981         then
 982                 echo "FAILED: Inotify test - incorrect state"
 983                 cat $PRS
 984                 ERR=1
 985         fi
 986         online_cpus
 987         echo member > cpuset.cpus.partition
 988         echo 0 > ../cgroup.procs
 989         if [[ $ERR -ne 0 ]]
 990         then
 991                 exit 1
 992         else
 993                 echo "Inotify test PASSED"
 994         fi
 995 }
 996
 997 trap cleanup 0 2 3 6
 998 run_state_test TEST_MATRIX
 999 test_isolated
1000 test_inotify
1001 echo "All tests PASSED."