nixos/tests/acme.nix

   1 { pkgs, lib, ... }: let
   2   commonConfig = ./common/acme/client;
   3
   4   dnsServerIP = nodes: nodes.dnsserver.networking.primaryIPAddress;
   5
   6   dnsScript = nodes: let
   7     dnsAddress = dnsServerIP nodes;
   8   in pkgs.writeShellScript "dns-hook.sh" ''
   9     set -euo pipefail
  10     echo '[INFO]' "[$2]" 'dns-hook.sh' $*
  11     if [ "$1" = "present" ]; then
  12       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'", "value": "'"$3"'"}' http://${dnsAddress}:8055/set-txt
  13     else
  14       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'"}' http://${dnsAddress}:8055/clear-txt
  15     fi
  16   '';
  17
  18   dnsConfig = nodes: {
  19     dnsProvider = "exec";
  20     dnsPropagationCheck = false;
  21     credentialsFile = pkgs.writeText "wildcard.env" ''
  22       EXEC_PATH=${dnsScript nodes}
  23       EXEC_POLLING_INTERVAL=1
  24       EXEC_PROPAGATION_TIMEOUT=1
  25       EXEC_SEQUENCE_INTERVAL=1
  26     '';
  27   };
  28
  29   documentRoot = pkgs.runCommand "docroot" {} ''
  30     mkdir -p "$out"
  31     echo hello world > "$out/index.html"
  32   '';
  33
  34   vhostBase = {
  35     forceSSL = true;
  36     locations."/".root = documentRoot;
  37   };
  38
  39   vhostBaseHttpd = {
  40     forceSSL = true;
  41     inherit documentRoot;
  42   };
  43
  44   simpleConfig = {
  45     security.acme = {
  46       certs."http.example.test" = {
  47         listenHTTP = ":80";
  48       };
  49     };
  50
  51     networking.firewall.allowedTCPPorts = [ 80 ];
  52   };
  53
  54   # Base specialisation config for testing general ACME features
  55   webserverBasicConfig = {
  56     services.nginx.enable = true;
  57     services.nginx.virtualHosts."a.example.test" = vhostBase // {
  58       enableACME = true;
  59     };
  60   };
  61
  62   # Generate specialisations for testing a web server
  63   mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
  64     baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
  65       {
  66         security.acme = {
  67           defaults = (dnsConfig nodes);
  68           # One manual wildcard cert
  69           certs."example.test" = {
  70             domain = "*.example.test";
  71           };
  72         };
  73
  74         users.users."${config.services."${server}".user}".extraGroups = ["acme"];
  75
  76         services."${server}" = {
  77           enable = true;
  78           virtualHosts = {
  79             # Run-of-the-mill vhost using HTTP-01 validation
  80             "${server}-http.example.test" = vhostBaseData // {
  81               serverAliases = [ "${server}-http-alias.example.test" ];
  82               enableACME = true;
  83             };
  84
  85             # Another which inherits the DNS-01 config
  86             "${server}-dns.example.test" = vhostBaseData // {
  87               serverAliases = [ "${server}-dns-alias.example.test" ];
  88               enableACME = true;
  89               # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
  90               # webroot + dnsProvider are mutually exclusive.
  91               acmeRoot = null;
  92             };
  93
  94             # One using the wildcard certificate
  95             "${server}-wildcard.example.test" = vhostBaseData // {
  96               serverAliases = [ "${server}-wildcard-alias.example.test" ];
  97               useACMEHost = "example.test";
  98             };
  99           };
 100         };
 101
 102         # Used to determine if service reload was triggered
 103         systemd.targets."test-renew-${server}" = {
 104           wants = [ "acme-${server}-http.example.test.service" ];
 105           after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
 106         };
 107       }
 108       specialConfig
 109       extraConfig
 110     ];
 111   in {
 112     "${server}".configuration = { nodes, config, ... }: baseConfig {
 113       inherit nodes config;
 114     };
 115
 116     # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
 117     "${server}-remove-alias".configuration = { nodes, config, ... }: baseConfig {
 118       inherit nodes config;
 119       specialConfig = {
 120         # Remove an alias, but create a standalone vhost in its place for testing.
 121         # This configuration results in certificate errors as useACMEHost does not imply
 122         # append extraDomains, and thus we can validate the SAN is removed.
 123         services."${server}" = {
 124           virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
 125           virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
 126             useACMEHost = "${server}-http.example.test";
 127           };
 128         };
 129       };
 130     };
 131
 132     # Test that the server reloads when only the acme configuration is changed.
 133     "${server}-change-acme-conf".configuration = { nodes, config, ... }: baseConfig {
 134       inherit nodes config;
 135       specialConfig = {
 136         security.acme.certs."${server}-http.example.test" = {
 137           keyType = "ec384";
 138           # Also test that postRun is exec'd as root
 139           postRun = "id | grep root";
 140         };
 141       };
 142     };
 143   };
 144
 145 in {
 146   name = "acme";
 147   meta = {
 148     maintainers = lib.teams.acme.members;
 149     # Hard timeout in seconds. Average run time is about 7 minutes.
 150     timeout = 1800;
 151   };
 152
 153   nodes = {
 154     # The fake ACME server which will respond to client requests
 155     acme = { nodes, ... }: {
 156       imports = [ ./common/acme/server ];
 157       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 158     };
 159
 160     # A fake DNS server which can be configured with records as desired
 161     # Used to test DNS-01 challenge
 162     dnsserver = { nodes, ... }: {
 163       networking.firewall.allowedTCPPorts = [ 8055 53 ];
 164       networking.firewall.allowedUDPPorts = [ 53 ];
 165       systemd.services.pebble-challtestsrv = {
 166         enable = true;
 167         description = "Pebble ACME challenge test server";
 168         wantedBy = [ "network.target" ];
 169         serviceConfig = {
 170           ExecStart = "${pkgs.pebble}/bin/pebble-challtestsrv -dns01 ':53' -defaultIPv6 '' -defaultIPv4 '${nodes.webserver.networking.primaryIPAddress}'";
 171           # Required to bind on privileged ports.
 172           AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
 173         };
 174       };
 175     };
 176
 177     # A web server which will be the node requesting certs
 178     webserver = { nodes, config, ... }: {
 179       imports = [ commonConfig ];
 180       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 181       networking.firewall.allowedTCPPorts = [ 80 443 ];
 182
 183       # OpenSSL will be used for more thorough certificate validation
 184       environment.systemPackages = [ pkgs.openssl ];
 185
 186       # Set log level to info so that we can see when the service is reloaded
 187       services.nginx.logError = "stderr info";
 188
 189       specialisation = {
 190         # Tests HTTP-01 verification using Lego's built-in web server
 191         http01lego.configuration = simpleConfig;
 192
 193         renew.configuration = lib.mkMerge [
 194           simpleConfig
 195           {
 196             # Pebble provides 5 year long certs,
 197             # needs to be higher than that to test renewal
 198             security.acme.certs."http.example.test".validMinDays = 9999;
 199           }
 200         ];
 201
 202         # Tests that account creds can be safely changed.
 203         accountchange.configuration = lib.mkMerge [
 204           simpleConfig
 205           {
 206             security.acme.certs."http.example.test".email = "admin@example.test";
 207           }
 208         ];
 209
 210         # First derivation used to test general ACME features
 211         general.configuration = { ... }: let
 212           caDomain = nodes.acme.test-support.acme.caDomain;
 213           email = config.security.acme.defaults.email;
 214           # Exit 99 to make it easier to track if this is the reason a renew failed
 215           accountCreateTester = ''
 216             test -e accounts/${caDomain}/${email}/account.json || exit 99
 217           '';
 218         in lib.mkMerge [
 219           webserverBasicConfig
 220           {
 221             # Used to test that account creation is collated into one service.
 222             # These should not run until after acme-finished-a.example.test.target
 223             systemd.services."b.example.test".preStart = accountCreateTester;
 224             systemd.services."c.example.test".preStart = accountCreateTester;
 225
 226             services.nginx.virtualHosts."b.example.test" = vhostBase // {
 227               enableACME = true;
 228             };
 229             services.nginx.virtualHosts."c.example.test" = vhostBase // {
 230               enableACME = true;
 231             };
 232           }
 233         ];
 234
 235         # Test OCSP Stapling
 236         ocsp-stapling.configuration = { ... }: lib.mkMerge [
 237           webserverBasicConfig
 238           {
 239             security.acme.certs."a.example.test".ocspMustStaple = true;
 240             services.nginx.virtualHosts."a.example.test" = {
 241               extraConfig = ''
 242                 ssl_stapling on;
 243                 ssl_stapling_verify on;
 244               '';
 245             };
 246           }
 247         ];
 248
 249         # Validate service relationships by adding a slow start service to nginx' wants.
 250         # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
 251         slow-startup.configuration = { ... }: lib.mkMerge [
 252           webserverBasicConfig
 253           {
 254             systemd.services.my-slow-service = {
 255               wantedBy = [ "multi-user.target" "nginx.service" ];
 256               before = [ "nginx.service" ];
 257               preStart = "sleep 5";
 258               script = "${pkgs.python3}/bin/python -m http.server";
 259             };
 260
 261             services.nginx.virtualHosts."slow.example.test" = {
 262               forceSSL = true;
 263               enableACME = true;
 264               locations."/".proxyPass = "http://localhost:8000";
 265             };
 266           }
 267         ];
 268
 269         # Test lego internal server (listenHTTP option)
 270         # Also tests useRoot option
 271         lego-server.configuration = { ... }: {
 272           security.acme.useRoot = true;
 273           security.acme.certs."lego.example.test" = {
 274             listenHTTP = ":80";
 275             group = "nginx";
 276           };
 277           services.nginx.enable = true;
 278           services.nginx.virtualHosts."lego.example.test" = {
 279             useACMEHost = "lego.example.test";
 280             onlySSL = true;
 281           };
 282         };
 283
 284       # Test compatibility with Caddy
 285       # It only supports useACMEHost, hence not using mkServerConfigs
 286       } // (let
 287         baseCaddyConfig = { nodes, config, ... }: {
 288           security.acme = {
 289             defaults = (dnsConfig nodes);
 290             # One manual wildcard cert
 291             certs."example.test" = {
 292               domain = "*.example.test";
 293             };
 294           };
 295
 296           users.users."${config.services.caddy.user}".extraGroups = ["acme"];
 297
 298           services.caddy = {
 299             enable = true;
 300             virtualHosts."a.exmaple.test" = {
 301               useACMEHost = "example.test";
 302               extraConfig = ''
 303                 root * ${documentRoot}
 304               '';
 305             };
 306           };
 307         };
 308       in {
 309         caddy.configuration = baseCaddyConfig;
 310
 311         # Test that the server reloads when only the acme configuration is changed.
 312         "caddy-change-acme-conf".configuration = { nodes, config, ... }: lib.mkMerge [
 313           (baseCaddyConfig {
 314             inherit nodes config;
 315           })
 316           {
 317             security.acme.certs."example.test" = {
 318               keyType = "ec384";
 319             };
 320           }
 321         ];
 322
 323       # Test compatibility with Nginx
 324       }) // (mkServerConfigs {
 325           server = "nginx";
 326           group = "nginx";
 327           vhostBaseData = vhostBase;
 328         })
 329
 330       # Test compatibility with Apache HTTPD
 331         // (mkServerConfigs {
 332           server = "httpd";
 333           group = "wwwrun";
 334           vhostBaseData = vhostBaseHttpd;
 335           extraConfig = {
 336             services.httpd.adminAddr = config.security.acme.defaults.email;
 337           };
 338         });
 339     };
 340
 341     # The client will be used to curl the webserver to validate configuration
 342     client = { nodes, ... }: {
 343       imports = [ commonConfig ];
 344       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 345
 346       # OpenSSL will be used for more thorough certificate validation
 347       environment.systemPackages = [ pkgs.openssl ];
 348     };
 349   };
 350
 351   testScript = { nodes, ... }:
 352     let
 353       caDomain = nodes.acme.test-support.acme.caDomain;
 354       newServerSystem = nodes.webserver.config.system.build.toplevel;
 355       switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test";
 356     in
 357     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
 358     # this is because a oneshot goes from inactive => activating => inactive, and never
 359     # reaches the active state. Targets do not have this issue.
 360     ''
 361       import time
 362
 363
 364       TOTAL_RETRIES = 20
 365
 366
 367       class BackoffTracker(object):
 368           delay = 1
 369           increment = 1
 370
 371           def handle_fail(self, retries, message) -> int:
 372               assert retries < TOTAL_RETRIES, message
 373
 374               print(f"Retrying in {self.delay}s, {retries + 1}/{TOTAL_RETRIES}")
 375               time.sleep(self.delay)
 376
 377               # Only increment after the first try
 378               if retries == 0:
 379                   self.delay += self.increment
 380                   self.increment *= 2
 381
 382               return retries + 1
 383
 384
 385       backoff = BackoffTracker()
 386
 387
 388       def switch_to(node, name):
 389           # On first switch, this will create a symlink to the current system so that we can
 390           # quickly switch between derivations
 391           root_specs = "/tmp/specialisation"
 392           node.execute(
 393             f"test -e {root_specs}"
 394             f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
 395           )
 396
 397           switcher_path = f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
 398           rc, _ = node.execute(f"test -e '{switcher_path}'")
 399           if rc > 0:
 400               switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 401
 402           node.succeed(
 403               f"{switcher_path} test"
 404           )
 405
 406
 407       # Ensures the issuer of our cert matches the chain
 408       # and matches the issuer we expect it to be.
 409       # It's a good validation to ensure the cert.pem and fullchain.pem
 410       # are not still selfsigned after verification
 411       def check_issuer(node, cert_name, issuer):
 412           for fname in ("cert.pem", "fullchain.pem"):
 413               actual_issuer = node.succeed(
 414                   f"openssl x509 -noout -issuer -in /var/lib/acme/{cert_name}/{fname}"
 415               ).partition("=")[2]
 416               print(f"{fname} issuer: {actual_issuer}")
 417               assert issuer.lower() in actual_issuer.lower()
 418
 419
 420       # Ensure cert comes before chain in fullchain.pem
 421       def check_fullchain(node, cert_name):
 422           subject_data = node.succeed(
 423               f"openssl crl2pkcs7 -nocrl -certfile /var/lib/acme/{cert_name}/fullchain.pem"
 424               " | openssl pkcs7 -print_certs -noout"
 425           )
 426           for line in subject_data.lower().split("\n"):
 427               if "subject" in line:
 428                   print(f"First subject in fullchain.pem: {line}")
 429                   assert cert_name.lower() in line
 430                   return
 431
 432           assert False
 433
 434
 435       def check_connection(node, domain, retries=0):
 436           result = node.succeed(
 437               "openssl s_client -brief -verify 2 -CAfile /tmp/ca.crt"
 438               f" -servername {domain} -connect {domain}:443 < /dev/null 2>&1"
 439           )
 440
 441           for line in result.lower().split("\n"):
 442               if "verification" in line and "error" in line:
 443                   retries = backoff.handle_fail(retries, f"Failed to connect to https://{domain}")
 444                   return check_connection(node, domain, retries)
 445
 446
 447       def check_connection_key_bits(node, domain, bits, retries=0):
 448           result = node.succeed(
 449               "openssl s_client -CAfile /tmp/ca.crt"
 450               f" -servername {domain} -connect {domain}:443 < /dev/null"
 451               " | openssl x509 -noout -text | grep -i Public-Key"
 452           )
 453           print("Key type:", result)
 454
 455           if bits not in result:
 456               retries = backoff.handle_fail(retries, f"Did not find expected number of bits ({bits}) in key")
 457               return check_connection_key_bits(node, domain, bits, retries)
 458
 459
 460       def check_stapling(node, domain, retries=0):
 461           # Pebble doesn't provide a full OCSP responder, so just check the URL
 462           result = node.succeed(
 463               "openssl s_client -CAfile /tmp/ca.crt"
 464               f" -servername {domain} -connect {domain}:443 < /dev/null"
 465               " | openssl x509 -noout -ocsp_uri"
 466           )
 467           print("OCSP Responder URL:", result)
 468
 469           if "${caDomain}:4002" not in result.lower():
 470               retries = backoff.handle_fail(retries, "OCSP Stapling check failed")
 471               return check_stapling(node, domain, retries)
 472
 473
 474       def download_ca_certs(node, retries=0):
 475           exit_code, _ = node.execute("curl https://${caDomain}:15000/roots/0 > /tmp/ca.crt")
 476           exit_code_2, _ = node.execute(
 477               "curl https://${caDomain}:15000/intermediate-keys/0 >> /tmp/ca.crt"
 478           )
 479
 480           if exit_code + exit_code_2 > 0:
 481               retries = backoff.handle_fail(retries, "Failed to connect to pebble to download root CA certs")
 482               return download_ca_certs(node, retries)
 483
 484
 485       start_all()
 486
 487       dnsserver.wait_for_unit("pebble-challtestsrv.service")
 488       client.wait_for_unit("default.target")
 489
 490       client.succeed(
 491           'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
 492       )
 493
 494       acme.wait_for_unit("network-online.target")
 495       acme.wait_for_unit("pebble.service")
 496
 497       download_ca_certs(client)
 498
 499       # Perform http-01 w/ lego test first
 500       with subtest("Can request certificate with Lego's built in web server"):
 501           switch_to(webserver, "http01lego")
 502           webserver.wait_for_unit("acme-finished-http.example.test.target")
 503           check_fullchain(webserver, "http.example.test")
 504           check_issuer(webserver, "http.example.test", "pebble")
 505
 506       # Perform renewal test
 507       with subtest("Can renew certificates when they expire"):
 508           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 509           switch_to(webserver, "renew")
 510           webserver.wait_for_unit("acme-finished-http.example.test.target")
 511           check_fullchain(webserver, "http.example.test")
 512           check_issuer(webserver, "http.example.test", "pebble")
 513           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 514           assert hash != hash_after
 515
 516       # Perform account change test
 517       with subtest("Handles email change correctly"):
 518           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 519           switch_to(webserver, "accountchange")
 520           webserver.wait_for_unit("acme-finished-http.example.test.target")
 521           check_fullchain(webserver, "http.example.test")
 522           check_issuer(webserver, "http.example.test", "pebble")
 523           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 524           # Has to do a full run to register account, which creates new certs.
 525           assert hash != hash_after
 526
 527       # Perform general tests
 528       switch_to(webserver, "general")
 529
 530       with subtest("Can request certificate with HTTP-01 challenge"):
 531           webserver.wait_for_unit("acme-finished-a.example.test.target")
 532           check_fullchain(webserver, "a.example.test")
 533           check_issuer(webserver, "a.example.test", "pebble")
 534           webserver.wait_for_unit("nginx.service")
 535           check_connection(client, "a.example.test")
 536
 537       with subtest("Runs 1 cert for account creation before others"):
 538           webserver.wait_for_unit("acme-finished-b.example.test.target")
 539           webserver.wait_for_unit("acme-finished-c.example.test.target")
 540           check_connection(client, "b.example.test")
 541           check_connection(client, "c.example.test")
 542
 543       with subtest("Certificates and accounts have safe + valid permissions"):
 544           # Nginx will set the group appropriately when enableACME is used
 545           group = "nginx"
 546           webserver.succeed(
 547               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 548           )
 549           webserver.succeed(
 550               f"test $(stat -L -c '%a %U %G' /var/lib/acme/.lego/a.example.test/**/a.example.test* | tee /dev/stderr | grep '600 acme {group}' | wc -l) -eq 4"
 551           )
 552           webserver.succeed(
 553               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test | tee /dev/stderr | grep '750 acme {group}' | wc -l) -eq 1"
 554           )
 555           webserver.succeed(
 556               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
 557           )
 558
 559       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
 560       with subtest("Can generate valid selfsigned certs"):
 561           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
 562           webserver.succeed("systemctl start acme-selfsigned-a.example.test.service")
 563           check_fullchain(webserver, "a.example.test")
 564           check_issuer(webserver, "a.example.test", "minica")
 565           # Check selfsigned permissions
 566           webserver.succeed(
 567               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 568           )
 569           # Will succeed if nginx can load the certs
 570           webserver.succeed("systemctl start nginx-config-reload.service")
 571
 572       with subtest("Correctly implements OCSP stapling"):
 573           switch_to(webserver, "ocsp-stapling")
 574           webserver.wait_for_unit("acme-finished-a.example.test.target")
 575           check_stapling(client, "a.example.test")
 576
 577       with subtest("Can request certificate with HTTP-01 using lego's internal web server"):
 578           switch_to(webserver, "lego-server")
 579           webserver.wait_for_unit("acme-finished-lego.example.test.target")
 580           webserver.wait_for_unit("nginx.service")
 581           webserver.succeed("echo HENLO && systemctl cat nginx.service")
 582           webserver.succeed("test \"$(stat -c '%U' /var/lib/acme/* | uniq)\" = \"root\"")
 583           check_connection(client, "a.example.test")
 584           check_connection(client, "lego.example.test")
 585
 586       with subtest("Can request certificate with HTTP-01 when nginx startup is delayed"):
 587           webserver.execute("systemctl stop nginx")
 588           switch_to(webserver, "slow-startup")
 589           webserver.wait_for_unit("acme-finished-slow.example.test.target")
 590           check_issuer(webserver, "slow.example.test", "pebble")
 591           webserver.wait_for_unit("nginx.service")
 592           check_connection(client, "slow.example.test")
 593
 594       with subtest("Works with caddy"):
 595           switch_to(webserver, "caddy")
 596           webserver.wait_for_unit("acme-finished-example.test.target")
 597           webserver.wait_for_unit("caddy.service")
 598           # FIXME reloading caddy is not sufficient to load new certs.
 599           # Restart it manually until this is fixed.
 600           webserver.succeed("systemctl restart caddy.service")
 601           check_connection(client, "a.example.test")
 602
 603       with subtest("security.acme changes reflect on caddy"):
 604           switch_to(webserver, "caddy-change-acme-conf")
 605           webserver.wait_for_unit("acme-finished-example.test.target")
 606           webserver.wait_for_unit("caddy.service")
 607           # FIXME reloading caddy is not sufficient to load new certs.
 608           # Restart it manually until this is fixed.
 609           webserver.succeed("systemctl restart caddy.service")
 610           check_connection_key_bits(client, "a.example.test", "384")
 611
 612       domains = ["http", "dns", "wildcard"]
 613       for server, logsrc in [
 614           ("nginx", "journalctl -n 30 -u nginx.service"),
 615           ("httpd", "tail -n 30 /var/log/httpd/*.log"),
 616       ]:
 617           wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
 618           with subtest(f"Works with {server}"):
 619               try:
 620                   switch_to(webserver, server)
 621                   # Skip wildcard domain for this check ([:-1])
 622                   for domain in domains[:-1]:
 623                       webserver.wait_for_unit(
 624                           f"acme-finished-{server}-{domain}.example.test.target"
 625                       )
 626               except Exception as err:
 627                   _, output = webserver.execute(
 628                       f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
 629                   )
 630                   print(output)
 631                   raise err
 632
 633               wait_for_server()
 634
 635               for domain in domains[:-1]:
 636                   check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
 637               for domain in domains:
 638                   check_connection(client, f"{server}-{domain}.example.test")
 639                   check_connection(client, f"{server}-{domain}-alias.example.test")
 640
 641           test_domain = f"{server}-{domains[0]}.example.test"
 642
 643           with subtest(f"Can reload {server} when timer triggers renewal"):
 644               # Switch to selfsigned first
 645               webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
 646               webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
 647               check_issuer(webserver, test_domain, "minica")
 648               webserver.succeed(f"systemctl start {server}-config-reload.service")
 649               webserver.succeed(f"systemctl start test-renew-{server}.target")
 650               check_issuer(webserver, test_domain, "pebble")
 651               check_connection(client, test_domain)
 652
 653           with subtest("Can remove an alias from a domain + cert is updated"):
 654               test_alias = f"{server}-{domains[0]}-alias.example.test"
 655               switch_to(webserver, f"{server}-remove-alias")
 656               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 657               wait_for_server()
 658               check_connection(client, test_domain)
 659               rc, _s = client.execute(
 660                   f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
 661                   " </dev/null 2>/dev/null | openssl x509 -noout -text"
 662                   f" | grep DNS: | grep {test_alias}"
 663               )
 664               assert rc > 0, "Removed extraDomainName was not removed from the cert"
 665
 666           with subtest("security.acme changes reflect on web server"):
 667               # Switch back to normal server config first, reset everything.
 668               switch_to(webserver, server)
 669               wait_for_server()
 670               switch_to(webserver, f"{server}-change-acme-conf")
 671               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 672               wait_for_server()
 673               check_connection_key_bits(client, test_domain, "384")
 674     '';
 675 }