test/units/TEST-55-OOMD.sh

   1 #!/usr/bin/env bash
   2 # SPDX-License-Identifier: LGPL-2.1-or-later
   3 set -eux
   4 set -o pipefail
   5
   6 # shellcheck source=test/units/test-control.sh
   7 . "$(dirname "$0")"/test-control.sh
   8 # shellcheck source=test/units/util.sh
   9 . "$(dirname "$0")"/util.sh
  10
  11 systemd-analyze log-level debug
  12
  13 # Ensure that the init.scope.d drop-in is applied on boot
  14 test "$(cat /sys/fs/cgroup/init.scope/memory.high)" != "max"
  15
  16 # Loose checks to ensure the environment has the necessary features for systemd-oomd
  17 [[ -e /proc/pressure ]] || echo "no PSI" >>/skipped
  18 [[ "$(get_cgroup_hierarchy)" == "unified" ]] || echo "no cgroupsv2" >>/skipped
  19 [[ -x /usr/lib/systemd/systemd-oomd ]] || echo "no oomd" >>/skipped
  20 if [[ -s /skipped ]]; then
  21     exit 77
  22 fi
  23
  24 # Activate swap file if we are in a VM
  25 if systemd-detect-virt --vm --quiet; then
  26     swapoff --all
  27     rm -f /swapfile
  28     if [[ "$(findmnt -n -o FSTYPE /)" == btrfs ]]; then
  29         btrfs filesystem mkswapfile -s 64M /swapfile
  30     else
  31         dd if=/dev/zero of=/swapfile bs=1M count=64
  32         chmod 0600 /swapfile
  33         mkswap /swapfile
  34     fi
  35
  36     swapon /swapfile
  37     swapon --show
  38 fi
  39
  40 # Configure oomd explicitly to avoid conflicts with distro dropins
  41 mkdir -p /run/systemd/oomd.conf.d/
  42 cat >/run/systemd/oomd.conf.d/99-oomd-test.conf <<EOF
  43 [OOM]
  44 DefaultMemoryPressureDurationSec=2s
  45 EOF
  46
  47 mkdir -p /run/systemd/system/-.slice.d/
  48 cat >/run/systemd/system/-.slice.d/99-oomd-test.conf <<EOF
  49 [Slice]
  50 ManagedOOMSwap=auto
  51 EOF
  52
  53 mkdir -p /run/systemd/system/user@.service.d/
  54 cat >/run/systemd/system/user@.service.d/99-oomd-test.conf <<EOF
  55 [Service]
  56 ManagedOOMMemoryPressure=auto
  57 ManagedOOMMemoryPressureLimit=0%
  58 EOF
  59
  60 mkdir -p /run/systemd/system/systemd-oomd.service.d/
  61 cat >/run/systemd/system/systemd-oomd.service.d/debug.conf <<EOF
  62 [Service]
  63 Environment=SYSTEMD_LOG_LEVEL=debug
  64 EOF
  65
  66 systemctl daemon-reload
  67
  68 # enable the service to ensure dbus-org.freedesktop.oom1.service exists
  69 # and D-Bus activation works
  70 systemctl enable systemd-oomd.service
  71
  72 # if oomd is already running for some reasons, then restart it to make sure the above settings to be applied
  73 if systemctl is-active systemd-oomd.service; then
  74     systemctl restart systemd-oomd.service
  75 fi
  76
  77 # Check if the oomd.conf drop-in config is loaded.
  78 assert_in 'Default Memory Pressure Duration: 2s' "$(oomctl)"
  79
  80 if [[ -v ASAN_OPTIONS || -v UBSAN_OPTIONS ]]; then
  81     # If we're running with sanitizers, sd-executor might pull in quite a significant chunk of shared
  82     # libraries, which in turn causes a lot of pressure that can put us in the front when sd-oomd decides to
  83     # go on a killing spree. This fact is exacerbated further on Arch Linux which ships unstripped gcc-libs,
  84     # so sd-executor pulls in over 30M of libs on startup. Let's make the MemoryHigh= limit a bit more
  85     # generous when running with sanitizers to make the test happy.
  86     systemctl edit --runtime --stdin --drop-in=99-MemoryHigh.conf TEST-55-OOMD-testchill.service <<EOF
  87 [Service]
  88 MemoryHigh=60M
  89 EOF
  90     # Do the same for the user instance as well
  91     mkdir -p /run/systemd/user/
  92     cp -rfv /run/systemd/system/TEST-55-OOMD-testchill.service.d/ /run/systemd/user/
  93 else
  94     # Ensure that we can start services even with a very low hard memory cap without oom-kills, but skip
  95     # under sanitizers as they balloon memory usage.
  96     systemd-run -t -p MemoryMax=10M -p MemorySwapMax=0 -p MemoryZSwapMax=0 /bin/true
  97 fi
  98
  99 test_basic() {
 100     local cgroup_path="${1:?}"
 101     shift
 102
 103     systemctl "$@" start TEST-55-OOMD-testchill.service
 104     systemctl "$@" status TEST-55-OOMD-testchill.service
 105     systemctl "$@" status TEST-55-OOMD-workload.slice
 106
 107     # Verify systemd-oomd is monitoring the expected units.
 108     timeout 1m bash -xec "until oomctl | grep -q -F 'Path: $cgroup_path'; do sleep 1; done"
 109     assert_in 'Memory Pressure Limit: 20.00%' \
 110               "$(oomctl | tac | sed -e '/Memory Pressure Monitored CGroups:/q' | tac | grep -A8 "Path: $cgroup_path")"
 111
 112     systemctl "$@" start TEST-55-OOMD-testbloat.service
 113
 114     # systemd-oomd watches for elevated pressure for 2 seconds before acting.
 115     # It can take time to build up pressure so either wait 2 minutes or for the service to fail.
 116     for _ in {0..59}; do
 117         if ! systemctl "$@" status TEST-55-OOMD-testbloat.service; then
 118             break
 119         fi
 120         oomctl
 121         sleep 2
 122     done
 123
 124     # testbloat should be killed and testchill should be fine
 125     if systemctl "$@" status TEST-55-OOMD-testbloat.service; then exit 42; fi
 126     if ! systemctl "$@" status TEST-55-OOMD-testchill.service; then exit 24; fi
 127
 128     systemctl "$@" kill --signal=KILL TEST-55-OOMD-testbloat.service || :
 129     systemctl "$@" stop TEST-55-OOMD-testbloat.service
 130     systemctl "$@" stop TEST-55-OOMD-testchill.service
 131     systemctl "$@" stop TEST-55-OOMD-workload.slice
 132 }
 133
 134 testcase_basic_system() {
 135     test_basic /TEST.slice/TEST-55.slice/TEST-55-OOMD.slice/TEST-55-OOMD-workload.slice
 136 }
 137
 138 testcase_basic_user() {
 139     # Make sure we also work correctly on user units.
 140     loginctl enable-linger testuser
 141
 142     test_basic "/user.slice/user-$(id -u testuser).slice/user@$(id -u testuser).service/TEST.slice/TEST-55.slice/TEST-55-OOMD.slice/TEST-55-OOMD-workload.slice" \
 143                --machine "testuser@.host" --user
 144
 145     loginctl disable-linger testuser
 146 }
 147
 148 testcase_preference_avoid() {
 149     # only run this portion of the test if we can set xattrs
 150     if ! cgroupfs_supports_user_xattrs; then
 151         echo "cgroup does not support user xattrs, skipping test for ManagedOOMPreference=avoid"
 152         return 0
 153     fi
 154
 155     mkdir -p /run/systemd/system/TEST-55-OOMD-testbloat.service.d/
 156     cat >/run/systemd/system/TEST-55-OOMD-testbloat.service.d/99-managed-oom-preference.conf <<EOF
 157 [Service]
 158 ManagedOOMPreference=avoid
 159 EOF
 160
 161     systemctl daemon-reload
 162     systemctl start TEST-55-OOMD-testchill.service
 163     systemctl start TEST-55-OOMD-testmunch.service
 164     systemctl start TEST-55-OOMD-testbloat.service
 165
 166     for _ in {0..59}; do
 167         if ! systemctl status TEST-55-OOMD-testmunch.service; then
 168             break
 169         fi
 170         oomctl
 171         sleep 2
 172     done
 173
 174     # testmunch should be killed since testbloat had the avoid xattr on it
 175     if ! systemctl status TEST-55-OOMD-testbloat.service; then exit 25; fi
 176     if systemctl status TEST-55-OOMD-testmunch.service; then exit 43; fi
 177     if ! systemctl status TEST-55-OOMD-testchill.service; then exit 24; fi
 178
 179     systemctl kill --signal=KILL TEST-55-OOMD-testbloat.service || :
 180     systemctl kill --signal=KILL TEST-55-OOMD-testmunch.service || :
 181     systemctl stop TEST-55-OOMD-testbloat.service
 182     systemctl stop TEST-55-OOMD-testmunch.service
 183     systemctl stop TEST-55-OOMD-testchill.service
 184     systemctl stop TEST-55-OOMD-workload.slice
 185
 186     # clean up overrides since test cases can be run in any order
 187     # and overrides shouldn't affect other tests
 188     rm -rf /run/systemd/system/TEST-55-OOMD-testbloat.service.d
 189     systemctl daemon-reload
 190 }
 191
 192 testcase_duration_analyze() {
 193     # Verify memory pressure duration is valid if >= 1 second
 194     cat <<EOF >/tmp/TEST-55-OOMD-valid-duration.service
 195 [Service]
 196 ExecStart=echo hello
 197 ManagedOOMMemoryPressureDurationSec=1s
 198 EOF
 199
 200     # Verify memory pressure duration is invalid if < 1 second
 201     cat <<EOF >/tmp/TEST-55-OOMD-invalid-duration.service
 202 [Service]
 203 ExecStart=echo hello
 204 ManagedOOMMemoryPressureDurationSec=0
 205 EOF
 206
 207     systemd-analyze --recursive-errors=no verify /tmp/TEST-55-OOMD-valid-duration.service
 208     (! systemd-analyze --recursive-errors=no verify /tmp/TEST-55-OOMD-invalid-duration.service)
 209
 210     rm -f /tmp/TEST-55-OOMD-valid-duration.service
 211     rm -f /tmp/TEST-55-OOMD-invalid-duration.service
 212 }
 213
 214 testcase_duration_override() {
 215     # Verify memory pressure duration can be overridden to non-zero values
 216     mkdir -p /run/systemd/system/TEST-55-OOMD-testmunch.service.d/
 217     cat >/run/systemd/system/TEST-55-OOMD-testmunch.service.d/99-duration-test.conf <<EOF
 218 [Service]
 219 ManagedOOMMemoryPressureDurationSec=3s
 220 ManagedOOMMemoryPressure=kill
 221 EOF
 222
 223     # Verify memory pressure duration will use default if set to empty
 224     mkdir -p /run/systemd/system/TEST-55-OOMD-testchill.service.d/
 225     cat >/run/systemd/system/TEST-55-OOMD-testchill.service.d/99-duration-test.conf <<EOF
 226 [Service]
 227 ManagedOOMMemoryPressureDurationSec=
 228 ManagedOOMMemoryPressure=kill
 229 EOF
 230
 231     systemctl daemon-reload
 232     systemctl start TEST-55-OOMD-testmunch.service
 233     systemctl start TEST-55-OOMD-testchill.service
 234
 235     timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-testmunch.service"; do sleep 1; done'
 236     oomctl | grep -A 2 "/TEST-55-OOMD-testmunch.service" | grep "Memory Pressure Duration: 3s"
 237
 238     timeout 1m bash -xec 'until oomctl | grep "/TEST-55-OOMD-testchill.service"; do sleep 1; done'
 239     oomctl | grep -A 2 "/TEST-55-OOMD-testchill.service" | grep "Memory Pressure Duration: 2s"
 240
 241     [[ "$(systemctl show -P ManagedOOMMemoryPressureDurationUSec TEST-55-OOMD-testmunch.service)" == "3s" ]]
 242     [[ "$(systemctl show -P ManagedOOMMemoryPressureDurationUSec TEST-55-OOMD-testchill.service)" == "[not set]" ]]
 243
 244     for _ in {0..59}; do
 245         if ! systemctl status TEST-55-OOMD-testmunch.service; then
 246             break
 247         fi
 248         oomctl
 249         sleep 2
 250     done
 251
 252     if systemctl status TEST-55-OOMD-testmunch.service; then exit 44; fi
 253     if ! systemctl status TEST-55-OOMD-testchill.service; then exit 23; fi
 254
 255     systemctl kill --signal=KILL TEST-55-OOMD-testmunch.service || :
 256     systemctl stop TEST-55-OOMD-testmunch.service
 257     systemctl stop TEST-55-OOMD-testchill.service
 258     systemctl stop TEST-55-OOMD-workload.slice
 259
 260     # clean up overrides since test cases can be run in any order
 261     # and overrides shouldn't affect other tests
 262     rm -rf /run/systemd/system/TEST-55-OOMD-testmunch.service.d
 263     rm -rf /run/systemd/system/TEST-55-OOMD-testchill.service.d
 264     systemctl daemon-reload
 265 }
 266
 267 testcase_reload() {
 268     # Check if the oomd.conf drop-in config is loaded.
 269     assert_in 'Swap Used Limit: 90.00%' "$(oomctl)"
 270     assert_in 'Default Memory Pressure Limit: 60.00%' "$(oomctl)"
 271     assert_in 'Default Memory Pressure Duration: 2s' "$(oomctl)"
 272
 273     # Test oomd reload
 274     mkdir -p /run/systemd/oomd.conf.d/
 275     {
 276         echo "[OOM]"
 277         echo "SwapUsedLimit=80%"
 278         echo "DefaultMemoryPressureLimit=55%"
 279         echo "DefaultMemoryPressureDurationSec=5s"
 280     } >/run/systemd/oomd.conf.d/99-oomd-test.conf
 281
 282     systemctl reload systemd-oomd.service
 283     assert_in 'Swap Used Limit: 80.00%' "$(oomctl)"
 284     assert_in 'Default Memory Pressure Limit: 55.00%' "$(oomctl)"
 285     assert_in 'Default Memory Pressure Duration: 5s' "$(oomctl)"
 286
 287     # Set back to default via reload
 288     mkdir -p /run/systemd/oomd.conf.d/
 289     {
 290         echo "[OOM]"
 291         echo "DefaultMemoryPressureDurationSec=2s"
 292     } >/run/systemd/oomd.conf.d/99-oomd-test.conf
 293
 294     systemctl reload systemd-oomd.service
 295
 296     assert_in 'Swap Used Limit: 90.00%' "$(oomctl)"
 297     assert_in 'Default Memory Pressure Limit: 60.00%' "$(oomctl)"
 298     assert_in 'Default Memory Pressure Duration: 2s' "$(oomctl)"
 299 }
 300
 301 run_testcases
 302
 303 systemd-analyze log-level info
 304
 305 touch /testok