test/units/TEST-82-SOFTREBOOT.sh

   1 #!/usr/bin/env bash
   2 # SPDX-License-Identifier: LGPL-2.1-or-later
   3 set -ex
   4 set -o pipefail
   5
   6 # shellcheck source=test/units/util.sh
   7 . "$(dirname "$0")"/util.sh
   8
   9 at_exit() {
  10     # Since the soft-reboot drops the enqueued end.service, we won't shutdown
  11     # the test VM if the test fails and have to wait for the watchdog to kill
  12     # us (which may take quite a long time). Let's just forcibly kill the machine
  13     # instead to save CI resources.
  14     if [[ $? -ne 0 ]]; then
  15         echo >&2 "Test failed, shutting down the machine..."
  16         systemctl poweroff -ff
  17     fi
  18 }
  19
  20 trap at_exit EXIT
  21
  22 # Because this test tests soft-reboot, we have to get rid of the symlink we put at
  23 # /run/nextroot to allow rebooting into the previous snapshot if the test fails for
  24 # the duration of the test. However, let's make sure we put the symlink back in place
  25 # if the test fails.
  26 if [[ -L /run/nextroot ]]; then
  27     at_error() {
  28         mountpoint -q /run/nextroot && umount -R /run/nextroot
  29         rm -rf /run/nextroot
  30         ln -sf /snapshot /run/nextroot
  31     }
  32
  33     trap at_error ERR
  34     rm -f /run/nextroot
  35 fi
  36
  37 systemd-analyze log-level debug
  38
  39 export SYSTEMD_LOG_LEVEL=debug
  40
  41 if [ -f /run/TEST-82-SOFTREBOOT.touch3 ]; then
  42     echo "This is the fourth boot!"
  43     systemd-notify --status="Fourth Boot"
  44
  45     test "$(busctl -j get-property org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager SoftRebootsCount | jq -r '.data')" -eq 3
  46
  47     rm /run/TEST-82-SOFTREBOOT.touch3
  48     mount
  49     rmdir /original-root /run/nextroot
  50
  51     # Check that the fdstore entry still exists
  52     test "$LISTEN_FDS" -eq 3
  53     read -r x <&5
  54     test "$x" = "oinkoink"
  55
  56     # Check that the surviving services are still around
  57     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive.service)" = "active"
  58     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive-argv.service)" = "active"
  59     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive-sigterm.service)" != "active"
  60     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive.service)" != "active"
  61
  62     [[ ! -e /run/credentials/TEST-82-SOFTREBOOT-nosurvive.service ]]
  63     assert_eq "$(cat /run/credentials/TEST-82-SOFTREBOOT-survive-argv.service/preserve)" "yay"
  64
  65     # There may be huge amount of pending messages in sockets. Processing them may cause journal rotation and
  66     # removal of old archived journal files. If a journal file is removed during journalctl reading it,
  67     # the command may fail. To mitigate such, sync before reading journals. Workaround for #32834.
  68     journalctl --sync
  69     # Check journals
  70     journalctl -o short-monotonic --no-hostname --grep '(will soft-reboot|KILL|corrupt)'
  71     assert_eq "$(journalctl -q -o short-monotonic -u systemd-journald.service --grep 'corrupt')" ""
  72
  73     # All succeeded, exit cleanly now
  74
  75 elif [ -f /run/TEST-82-SOFTREBOOT.touch2 ]; then
  76     echo "This is the third boot!"
  77     systemd-notify --status="Third Boot"
  78
  79     test "$(busctl -j get-property org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager SoftRebootsCount | jq -r '.data')" -eq 2
  80
  81     rm /run/TEST-82-SOFTREBOOT.touch2
  82
  83     # Check that the fdstore entry still exists
  84     test "$LISTEN_FDS" -eq 2
  85     read -r x <&4
  86     test "$x" = "miaumiau"
  87
  88     # Upload another entry
  89     T="/dev/shm/fdstore.$RANDOM"
  90     echo "oinkoink" >"$T"
  91     systemd-notify --fd=3 --pid=parent 3<"$T"
  92     rm "$T"
  93
  94     # Check that the surviving services are still around
  95     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive.service)" = "active"
  96     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive-argv.service)" = "active"
  97     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive-sigterm.service)" != "active"
  98     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive.service)" != "active"
  99
 100     # Test that we really are in the new overlayfs root fs
 101     read -r x </lower
 102     test "$x" = "miep"
 103     cmp /etc/os-release /run/systemd/propagate/.os-release-stage/os-release
 104     grep -q MARKER=1 /etc/os-release
 105
 106     # Switch back to the original root, away from the overlayfs
 107     mount --bind /original-root /run/nextroot
 108     mount
 109
 110     # Restart the unit that is not supposed to survive
 111     systemd-run --collect --service-type=exec --unit=TEST-82-SOFTREBOOT-nosurvive.service sleep infinity
 112
 113     # Now issue the soft reboot. We should be right back soon.
 114     touch /run/TEST-82-SOFTREBOOT.touch3
 115     systemctl --no-block soft-reboot
 116
 117     # Now block until the soft-boot killing spree kills us
 118     exec sleep infinity
 119
 120 elif [ -f /run/TEST-82-SOFTREBOOT.touch ]; then
 121     echo "This is the second boot!"
 122     systemd-notify --status="Second Boot"
 123
 124     test "$(busctl -j get-property org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager SoftRebootsCount | jq -r '.data')" -eq 1
 125
 126     # Clean up what we created earlier
 127     rm /run/TEST-82-SOFTREBOOT.touch
 128
 129     # Check that the fdstore entry still exists
 130     test "$LISTEN_FDS" -eq 1
 131     read -r x <&3
 132     test "$x" = "wuffwuff"
 133
 134     # Check that we got a PrepareForShutdownWithMetadata signal with the right type
 135     cat /run/TEST-82-SOFTREBOOT.signal
 136     test "$(jq -r '.payload.data[1].type.data' </run/TEST-82-SOFTREBOOT.signal)" = "soft-reboot"
 137
 138     # Check that the system credentials survived the soft reboot.
 139     test "$(systemd-creds cat --system kernelcmdlinecred)" = "uff"
 140
 141     # Upload another entry
 142     T="/dev/shm/fdstore.$RANDOM"
 143     echo "miaumiau" >"$T"
 144     systemd-notify --fd=3 --pid=parent 3<"$T"
 145     rm "$T"
 146
 147     # Check that the surviving services are still around
 148     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive.service)" = "active"
 149     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-survive-argv.service)" = "active"
 150     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive-sigterm.service)" != "active"
 151     test "$(systemctl show -P ActiveState TEST-82-SOFTREBOOT-nosurvive.service)" != "active"
 152
 153     # This time we test the /run/nextroot/ root switching logic. (We synthesize a new rootfs from the old via overlayfs)
 154     mkdir -p /run/nextroot /tmp/nextroot-lower /original-root
 155     mount -t tmpfs tmpfs /tmp/nextroot-lower
 156     echo miep >/tmp/nextroot-lower/lower
 157
 158     # Copy os-release away, so that we can manipulate it and check that it is updated in the propagate
 159     # directory across soft reboots. Try to cover corner cases by truncating it.
 160     mkdir -p /tmp/nextroot-lower/etc
 161     grep ID /etc/os-release >/tmp/nextroot-lower/etc/os-release
 162     echo MARKER=1 >>/tmp/nextroot-lower/etc/os-release
 163     cmp /etc/os-release /run/systemd/propagate/.os-release-stage/os-release
 164     (! grep -q MARKER=1 /etc/os-release)
 165
 166     mount -t overlay nextroot /run/nextroot -o lowerdir=/tmp/nextroot-lower:/,ro
 167
 168     # Bind our current root into the target so that we later can return to it
 169     mount --bind / /run/nextroot/original-root
 170
 171     # Restart the unit that is not supposed to survive
 172     systemd-run --collect --service-type=exec --unit=TEST-82-SOFTREBOOT-nosurvive.service sleep infinity
 173
 174     # Now ensure there are no naming clashes and a bunch of transient units all succeed
 175     for _ in $(seq 1 25); do
 176         systemd-run --wait true
 177     done
 178
 179     # Now issue the soft reboot. We should be right back soon. Given /run/nextroot exists, we should
 180     # automatically do a softreboot instead of normal reboot.
 181     touch /run/TEST-82-SOFTREBOOT.touch2
 182     systemctl --no-block reboot
 183
 184     # Now block until the soft-boot killing spree kills us
 185     exec sleep infinity
 186 else
 187     # This is the first boot
 188     systemd-notify --status="First Boot"
 189
 190     test "$(busctl -j get-property org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager SoftRebootsCount | jq -r '.data')" -eq 0
 191
 192     # Let's upload an fd to the fdstore, so that we can verify fdstore passing works correctly
 193     T="/dev/shm/fdstore.$RANDOM"
 194     echo "wuffwuff" >"$T"
 195     systemd-notify --fd=3 --pid=parent 3<"$T"
 196     rm "$T"
 197
 198     survive_sigterm="/dev/shm/survive-sigterm-$RANDOM.sh"
 199     cat >"$survive_sigterm" <<EOF
 200 #!/bin/bash
 201 trap "" TERM
 202 systemd-notify --ready
 203 rm "$survive_sigterm"
 204 exec sleep infinity
 205 EOF
 206     chmod +x "$survive_sigterm"
 207
 208     survive_argv="/dev/shm/survive-argv-$RANDOM.sh"
 209     cat >"$survive_argv" <<EOF
 210 #!/bin/bash
 211 systemd-notify --ready
 212 rm "$survive_argv"
 213 exec -a @sleep sleep infinity
 214 EOF
 215     chmod +x "$survive_argv"
 216     # This sets DefaultDependencies=no so that they remain running until the very end, and
 217     # IgnoreOnIsolate=yes so that they aren't stopped via the "testsuite.target" isolation we do on next boot,
 218     # and will be killed by the final sigterm/sigkill spree.
 219     systemd-run --collect --service-type=notify -p DefaultDependencies=no -p IgnoreOnIsolate=yes --unit=TEST-82-SOFTREBOOT-nosurvive-sigterm.service "$survive_sigterm"
 220     systemd-run --collect --service-type=exec -p DefaultDependencies=no -p IgnoreOnIsolate=yes -p SetCredential=gone:hoge --unit=TEST-82-SOFTREBOOT-nosurvive.service sleep infinity
 221
 222     # Ensure that the unit doesn't get deactivated by dependencies on the source file. Given it's a verity
 223     # image that is already open, even if the tmpfs with the image goes away, the file will be pinned by the
 224     # kernel and will keep working.
 225     cp /usr/share/minimal_0.* /tmp/
 226
 227     # Configure these transient units to survive the soft reboot - they will not conflict with shutdown.target
 228     # and it will be ignored on the isolate that happens in the next boot. The first will use argv[0][0] =
 229     # '@', and the second will use SurviveFinalKillSignal=yes. Both should survive.
 230     # By writing to stdout, which is connected to the journal, we also ensure logging doesn't break across
 231     # soft reboots due to journald being temporarily stopped.
 232     systemd-run --service-type=notify --unit=TEST-82-SOFTREBOOT-survive-argv.service \
 233         --property SurviveFinalKillSignal=no \
 234         --property IgnoreOnIsolate=yes \
 235         --property DefaultDependencies=no \
 236         --property After=basic.target \
 237         --property "Conflicts=reboot.target kexec.target poweroff.target halt.target emergency.target rescue.target" \
 238         --property "Before=reboot.target kexec.target poweroff.target halt.target emergency.target rescue.target" \
 239         --property SetCredential=preserve:yay \
 240          "$survive_argv"
 241     # shellcheck disable=SC2016
 242     systemd-run --service-type=exec --unit=TEST-82-SOFTREBOOT-survive.service \
 243         --property TemporaryFileSystem="/run /tmp /var" \
 244         --property RootImage=/tmp/minimal_0.raw \
 245         --property SurviveFinalKillSignal=yes \
 246         --property IgnoreOnIsolate=yes \
 247         --property DefaultDependencies=no \
 248         --property After=basic.target \
 249         --property "Conflicts=reboot.target kexec.target poweroff.target halt.target emergency.target rescue.target" \
 250         --property "Before=reboot.target kexec.target poweroff.target halt.target emergency.target rescue.target" \
 251         bash -c 'count=0; while echo "$count"; do count=$[$count +1]; sleep 1; done'
 252
 253     # Check that we can set up an inhibitor, and that busctl monitor sees the
 254     # PrepareForShutdownWithMetadata signal and that it says 'soft-reboot'.
 255     systemd-run --unit busctl.service --service-type=exec --property StandardOutput=file:/run/TEST-82-SOFTREBOOT.signal \
 256         busctl monitor --json=pretty --match 'sender=org.freedesktop.login1,path=/org/freedesktop/login1,interface=org.freedesktop.login1.Manager,member=PrepareForShutdownWithMetadata,type=signal'
 257     systemd-run --unit inhibit.service --service-type=exec \
 258         systemd-inhibit --what=shutdown --who=test --why=test --mode=delay \
 259             sleep infinity
 260
 261     # Enqueue a bunch of failing units to try and trigger the transient name clash that happens due to D-Bus
 262     # being restarted and the "unique" bus IDs not being unique across restarts
 263     for _ in $(seq 1 25); do
 264         # Use --wait to ensure we connect to the system bus instead of the private bus (otherwise a UUID is
 265         # used instead of the bus ID)
 266         systemd-run --wait false || true
 267     done
 268
 269     # Now issue the soft reboot. We should be right back soon.
 270     touch /run/TEST-82-SOFTREBOOT.touch
 271     systemctl --no-block --check-inhibitors=yes soft-reboot
 272
 273     # Ensure the property works too
 274     type="$(busctl --json=short get-property org.freedesktop.login1 /org/freedesktop/login1 org.freedesktop.login1.Manager PreparingForShutdownWithMetadata | jq -r '.data.type.data')"
 275     test "$type" = "soft-reboot"
 276
 277     # Now block until the soft-boot killing spree kills us
 278     exec sleep infinity
 279 fi
 280
 281 systemd-analyze log-level info
 282
 283 touch /testok
 284 systemctl --no-block exit 123