nixos/tests/acme.nix

   1 { pkgs, lib, ... }: let
   2   commonConfig = ./common/acme/client;
   3
   4   dnsServerIP = nodes: nodes.dnsserver.networking.primaryIPAddress;
   5
   6   dnsScript = nodes: let
   7     dnsAddress = dnsServerIP nodes;
   8   in pkgs.writeShellScript "dns-hook.sh" ''
   9     set -euo pipefail
  10     echo '[INFO]' "[$2]" 'dns-hook.sh' $*
  11     if [ "$1" = "present" ]; then
  12       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'", "value": "'"$3"'"}' http://${dnsAddress}:8055/set-txt
  13     else
  14       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'"}' http://${dnsAddress}:8055/clear-txt
  15     fi
  16   '';
  17
  18   dnsConfig = nodes: {
  19     dnsProvider = "exec";
  20     dnsPropagationCheck = false;
  21     credentialsFile = pkgs.writeText "wildcard.env" ''
  22       EXEC_PATH=${dnsScript nodes}
  23       EXEC_POLLING_INTERVAL=1
  24       EXEC_PROPAGATION_TIMEOUT=1
  25       EXEC_SEQUENCE_INTERVAL=1
  26     '';
  27   };
  28
  29   documentRoot = pkgs.runCommand "docroot" {} ''
  30     mkdir -p "$out"
  31     echo hello world > "$out/index.html"
  32   '';
  33
  34   vhostBase = {
  35     forceSSL = true;
  36     locations."/".root = documentRoot;
  37   };
  38
  39   vhostBaseHttpd = {
  40     forceSSL = true;
  41     inherit documentRoot;
  42   };
  43
  44   simpleConfig = {
  45     security.acme = {
  46       certs."http.example.test" = {
  47         listenHTTP = ":80";
  48       };
  49     };
  50
  51     networking.firewall.allowedTCPPorts = [ 80 ];
  52   };
  53
  54   # Base specialisation config for testing general ACME features
  55   webserverBasicConfig = {
  56     services.nginx.enable = true;
  57     services.nginx.virtualHosts."a.example.test" = vhostBase // {
  58       enableACME = true;
  59     };
  60   };
  61
  62   # Generate specialisations for testing a web server
  63   mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
  64     baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
  65       {
  66         security.acme = {
  67           defaults = (dnsConfig nodes);
  68           # One manual wildcard cert
  69           certs."example.test" = {
  70             domain = "*.example.test";
  71           };
  72         };
  73
  74         users.users."${config.services."${server}".user}".extraGroups = ["acme"];
  75
  76         services."${server}" = {
  77           enable = true;
  78           virtualHosts = {
  79             # Run-of-the-mill vhost using HTTP-01 validation
  80             "${server}-http.example.test" = vhostBaseData // {
  81               serverAliases = [ "${server}-http-alias.example.test" ];
  82               enableACME = true;
  83             };
  84
  85             # Another which inherits the DNS-01 config
  86             "${server}-dns.example.test" = vhostBaseData // {
  87               serverAliases = [ "${server}-dns-alias.example.test" ];
  88               enableACME = true;
  89               # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
  90               # webroot + dnsProvider are mutually exclusive.
  91               acmeRoot = null;
  92             };
  93
  94             # One using the wildcard certificate
  95             "${server}-wildcard.example.test" = vhostBaseData // {
  96               serverAliases = [ "${server}-wildcard-alias.example.test" ];
  97               useACMEHost = "example.test";
  98             };
  99           };
 100         };
 101
 102         # Used to determine if service reload was triggered
 103         systemd.targets."test-renew-${server}" = {
 104           wants = [ "acme-${server}-http.example.test.service" ];
 105           after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
 106         };
 107       }
 108       specialConfig
 109       extraConfig
 110     ];
 111   in {
 112     "${server}".configuration = { nodes, config, ... }: baseConfig {
 113       inherit nodes config;
 114     };
 115
 116     # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
 117     "${server}-remove-alias".configuration = { nodes, config, ... }: baseConfig {
 118       inherit nodes config;
 119       specialConfig = {
 120         # Remove an alias, but create a standalone vhost in its place for testing.
 121         # This configuration results in certificate errors as useACMEHost does not imply
 122         # append extraDomains, and thus we can validate the SAN is removed.
 123         services."${server}" = {
 124           virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
 125           virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
 126             useACMEHost = "${server}-http.example.test";
 127           };
 128         };
 129       };
 130     };
 131
 132     # Test that the server reloads when only the acme configuration is changed.
 133     "${server}-change-acme-conf".configuration = { nodes, config, ... }: baseConfig {
 134       inherit nodes config;
 135       specialConfig = {
 136         security.acme.certs."${server}-http.example.test" = {
 137           keyType = "ec384";
 138           # Also test that postRun is exec'd as root
 139           postRun = "id | grep root";
 140         };
 141       };
 142     };
 143   };
 144
 145 in {
 146   name = "acme";
 147   meta.maintainers = lib.teams.acme.members;
 148
 149   nodes = {
 150     # The fake ACME server which will respond to client requests
 151     acme = { nodes, ... }: {
 152       imports = [ ./common/acme/server ];
 153       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 154     };
 155
 156     # A fake DNS server which can be configured with records as desired
 157     # Used to test DNS-01 challenge
 158     dnsserver = { nodes, ... }: {
 159       networking.firewall.allowedTCPPorts = [ 8055 53 ];
 160       networking.firewall.allowedUDPPorts = [ 53 ];
 161       systemd.services.pebble-challtestsrv = {
 162         enable = true;
 163         description = "Pebble ACME challenge test server";
 164         wantedBy = [ "network.target" ];
 165         serviceConfig = {
 166           ExecStart = "${pkgs.pebble}/bin/pebble-challtestsrv -dns01 ':53' -defaultIPv6 '' -defaultIPv4 '${nodes.webserver.networking.primaryIPAddress}'";
 167           # Required to bind on privileged ports.
 168           AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
 169         };
 170       };
 171     };
 172
 173     # A web server which will be the node requesting certs
 174     webserver = { nodes, config, ... }: {
 175       imports = [ commonConfig ];
 176       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 177       networking.firewall.allowedTCPPorts = [ 80 443 ];
 178
 179       # OpenSSL will be used for more thorough certificate validation
 180       environment.systemPackages = [ pkgs.openssl ];
 181
 182       # Set log level to info so that we can see when the service is reloaded
 183       services.nginx.logError = "stderr info";
 184
 185       specialisation = {
 186         # Tests HTTP-01 verification using Lego's built-in web server
 187         http01lego.configuration = simpleConfig;
 188
 189         renew.configuration = lib.mkMerge [
 190           simpleConfig
 191           {
 192             # Pebble provides 5 year long certs,
 193             # needs to be higher than that to test renewal
 194             security.acme.certs."http.example.test".validMinDays = 9999;
 195           }
 196         ];
 197
 198         # Tests that account creds can be safely changed.
 199         accountchange.configuration = lib.mkMerge [
 200           simpleConfig
 201           {
 202             security.acme.certs."http.example.test".email = "admin@example.test";
 203           }
 204         ];
 205
 206         # First derivation used to test general ACME features
 207         general.configuration = { ... }: let
 208           caDomain = nodes.acme.test-support.acme.caDomain;
 209           email = config.security.acme.defaults.email;
 210           # Exit 99 to make it easier to track if this is the reason a renew failed
 211           accountCreateTester = ''
 212             test -e accounts/${caDomain}/${email}/account.json || exit 99
 213           '';
 214         in lib.mkMerge [
 215           webserverBasicConfig
 216           {
 217             # Used to test that account creation is collated into one service.
 218             # These should not run until after acme-finished-a.example.test.target
 219             systemd.services."b.example.test".preStart = accountCreateTester;
 220             systemd.services."c.example.test".preStart = accountCreateTester;
 221
 222             services.nginx.virtualHosts."b.example.test" = vhostBase // {
 223               enableACME = true;
 224             };
 225             services.nginx.virtualHosts."c.example.test" = vhostBase // {
 226               enableACME = true;
 227             };
 228           }
 229         ];
 230
 231         # Test OCSP Stapling
 232         ocsp-stapling.configuration = { ... }: lib.mkMerge [
 233           webserverBasicConfig
 234           {
 235             security.acme.certs."a.example.test".ocspMustStaple = true;
 236             services.nginx.virtualHosts."a.example.test" = {
 237               extraConfig = ''
 238                 ssl_stapling on;
 239                 ssl_stapling_verify on;
 240               '';
 241             };
 242           }
 243         ];
 244
 245         # Validate service relationships by adding a slow start service to nginx' wants.
 246         # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
 247         slow-startup.configuration = { ... }: lib.mkMerge [
 248           webserverBasicConfig
 249           {
 250             systemd.services.my-slow-service = {
 251               wantedBy = [ "multi-user.target" "nginx.service" ];
 252               before = [ "nginx.service" ];
 253               preStart = "sleep 5";
 254               script = "${pkgs.python3}/bin/python -m http.server";
 255             };
 256
 257             services.nginx.virtualHosts."slow.example.test" = {
 258               forceSSL = true;
 259               enableACME = true;
 260               locations."/".proxyPass = "http://localhost:8000";
 261             };
 262           }
 263         ];
 264
 265         # Test lego internal server (listenHTTP option)
 266         # Also tests useRoot option
 267         lego-server.configuration = { ... }: {
 268           security.acme.useRoot = true;
 269           security.acme.certs."lego.example.test" = {
 270             listenHTTP = ":80";
 271             group = "nginx";
 272           };
 273           services.nginx.enable = true;
 274           services.nginx.virtualHosts."lego.example.test" = {
 275             useACMEHost = "lego.example.test";
 276             onlySSL = true;
 277           };
 278         };
 279
 280       # Test compatiblity with Caddy
 281       # It only supports useACMEHost, hence not using mkServerConfigs
 282       } // (let
 283         baseCaddyConfig = { nodes, config, ... }: {
 284           security.acme = {
 285             defaults = (dnsConfig nodes);
 286             # One manual wildcard cert
 287             certs."example.test" = {
 288               domain = "*.example.test";
 289             };
 290           };
 291
 292           users.users."${config.services.caddy.user}".extraGroups = ["acme"];
 293
 294           services.caddy = {
 295             enable = true;
 296             virtualHosts."a.exmaple.test" = {
 297               useACMEHost = "example.test";
 298               extraConfig = ''
 299                 root * ${documentRoot}
 300               '';
 301             };
 302           };
 303         };
 304       in {
 305         caddy.configuration = baseCaddyConfig;
 306
 307         # Test that the server reloads when only the acme configuration is changed.
 308         "caddy-change-acme-conf".configuration = { nodes, config, ... }: lib.mkMerge [
 309           (baseCaddyConfig {
 310             inherit nodes config;
 311           })
 312           {
 313             security.acme.certs."example.test" = {
 314               keyType = "ec384";
 315             };
 316           }
 317         ];
 318
 319       # Test compatibility with Nginx
 320       }) // (mkServerConfigs {
 321           server = "nginx";
 322           group = "nginx";
 323           vhostBaseData = vhostBase;
 324         })
 325
 326       # Test compatibility with Apache HTTPD
 327         // (mkServerConfigs {
 328           server = "httpd";
 329           group = "wwwrun";
 330           vhostBaseData = vhostBaseHttpd;
 331           extraConfig = {
 332             services.httpd.adminAddr = config.security.acme.defaults.email;
 333           };
 334         });
 335     };
 336
 337     # The client will be used to curl the webserver to validate configuration
 338     client = { nodes, ... }: {
 339       imports = [ commonConfig ];
 340       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 341
 342       # OpenSSL will be used for more thorough certificate validation
 343       environment.systemPackages = [ pkgs.openssl ];
 344     };
 345   };
 346
 347   testScript = { nodes, ... }:
 348     let
 349       caDomain = nodes.acme.test-support.acme.caDomain;
 350       newServerSystem = nodes.webserver.config.system.build.toplevel;
 351       switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test";
 352     in
 353     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
 354     # this is because a oneshot goes from inactive => activating => inactive, and never
 355     # reaches the active state. Targets do not have this issue.
 356     ''
 357       import time
 358
 359
 360       def switch_to(node, name):
 361           # On first switch, this will create a symlink to the current system so that we can
 362           # quickly switch between derivations
 363           root_specs = "/tmp/specialisation"
 364           node.execute(
 365             f"test -e {root_specs}"
 366             f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
 367           )
 368
 369           switcher_path = f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
 370           rc, _ = node.execute(f"test -e '{switcher_path}'")
 371           if rc > 0:
 372               switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 373
 374           node.succeed(
 375               f"{switcher_path} test"
 376           )
 377
 378
 379       # Ensures the issuer of our cert matches the chain
 380       # and matches the issuer we expect it to be.
 381       # It's a good validation to ensure the cert.pem and fullchain.pem
 382       # are not still selfsigned afer verification
 383       def check_issuer(node, cert_name, issuer):
 384           for fname in ("cert.pem", "fullchain.pem"):
 385               actual_issuer = node.succeed(
 386                   f"openssl x509 -noout -issuer -in /var/lib/acme/{cert_name}/{fname}"
 387               ).partition("=")[2]
 388               print(f"{fname} issuer: {actual_issuer}")
 389               assert issuer.lower() in actual_issuer.lower()
 390
 391
 392       # Ensure cert comes before chain in fullchain.pem
 393       def check_fullchain(node, cert_name):
 394           subject_data = node.succeed(
 395               f"openssl crl2pkcs7 -nocrl -certfile /var/lib/acme/{cert_name}/fullchain.pem"
 396               " | openssl pkcs7 -print_certs -noout"
 397           )
 398           for line in subject_data.lower().split("\n"):
 399               if "subject" in line:
 400                   print(f"First subject in fullchain.pem: {line}")
 401                   assert cert_name.lower() in line
 402                   return
 403
 404           assert False
 405
 406
 407       def check_connection(node, domain, retries=3):
 408           assert retries >= 0, f"Failed to connect to https://{domain}"
 409
 410           result = node.succeed(
 411               "openssl s_client -brief -verify 2 -CAfile /tmp/ca.crt"
 412               f" -servername {domain} -connect {domain}:443 < /dev/null 2>&1"
 413           )
 414
 415           for line in result.lower().split("\n"):
 416               if "verification" in line and "error" in line:
 417                   time.sleep(3)
 418                   return check_connection(node, domain, retries - 1)
 419
 420
 421       def check_connection_key_bits(node, domain, bits, retries=3):
 422           assert retries >= 0, f"Did not find expected number of bits ({bits}) in key"
 423
 424           result = node.succeed(
 425               "openssl s_client -CAfile /tmp/ca.crt"
 426               f" -servername {domain} -connect {domain}:443 < /dev/null"
 427               " | openssl x509 -noout -text | grep -i Public-Key"
 428           )
 429           print("Key type:", result)
 430
 431           if bits not in result:
 432               time.sleep(3)
 433               return check_connection_key_bits(node, domain, bits, retries - 1)
 434
 435
 436       def check_stapling(node, domain, retries=3):
 437           assert retries >= 0, "OCSP Stapling check failed"
 438
 439           # Pebble doesn't provide a full OCSP responder, so just check the URL
 440           result = node.succeed(
 441               "openssl s_client -CAfile /tmp/ca.crt"
 442               f" -servername {domain} -connect {domain}:443 < /dev/null"
 443               " | openssl x509 -noout -ocsp_uri"
 444           )
 445           print("OCSP Responder URL:", result)
 446
 447           if "${caDomain}:4002" not in result.lower():
 448               time.sleep(3)
 449               return check_stapling(node, domain, retries - 1)
 450
 451
 452       def download_ca_certs(node, retries=5):
 453           assert retries >= 0, "Failed to connect to pebble to download root CA certs"
 454
 455           exit_code, _ = node.execute("curl https://${caDomain}:15000/roots/0 > /tmp/ca.crt")
 456           exit_code_2, _ = node.execute(
 457               "curl https://${caDomain}:15000/intermediate-keys/0 >> /tmp/ca.crt"
 458           )
 459
 460           if exit_code + exit_code_2 > 0:
 461               time.sleep(3)
 462               return download_ca_certs(node, retries - 1)
 463
 464
 465       start_all()
 466
 467       dnsserver.wait_for_unit("pebble-challtestsrv.service")
 468       client.wait_for_unit("default.target")
 469
 470       client.succeed(
 471           'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
 472       )
 473
 474       acme.wait_for_unit("network-online.target")
 475       acme.wait_for_unit("pebble.service")
 476
 477       download_ca_certs(client)
 478
 479       # Perform http-01 w/ lego test first
 480       with subtest("Can request certificate with Lego's built in web server"):
 481           switch_to(webserver, "http01lego")
 482           webserver.wait_for_unit("acme-finished-http.example.test.target")
 483           check_fullchain(webserver, "http.example.test")
 484           check_issuer(webserver, "http.example.test", "pebble")
 485
 486       # Perform renewal test
 487       with subtest("Can renew certificates when they expire"):
 488           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 489           switch_to(webserver, "renew")
 490           webserver.wait_for_unit("acme-finished-http.example.test.target")
 491           check_fullchain(webserver, "http.example.test")
 492           check_issuer(webserver, "http.example.test", "pebble")
 493           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 494           assert hash != hash_after
 495
 496       # Perform account change test
 497       with subtest("Handles email change correctly"):
 498           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 499           switch_to(webserver, "accountchange")
 500           webserver.wait_for_unit("acme-finished-http.example.test.target")
 501           check_fullchain(webserver, "http.example.test")
 502           check_issuer(webserver, "http.example.test", "pebble")
 503           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 504           # Has to do a full run to register account, which creates new certs.
 505           assert hash != hash_after
 506
 507       # Perform general tests
 508       switch_to(webserver, "general")
 509
 510       with subtest("Can request certificate with HTTP-01 challenge"):
 511           webserver.wait_for_unit("acme-finished-a.example.test.target")
 512           check_fullchain(webserver, "a.example.test")
 513           check_issuer(webserver, "a.example.test", "pebble")
 514           webserver.wait_for_unit("nginx.service")
 515           check_connection(client, "a.example.test")
 516
 517       with subtest("Runs 1 cert for account creation before others"):
 518           webserver.wait_for_unit("acme-finished-b.example.test.target")
 519           webserver.wait_for_unit("acme-finished-c.example.test.target")
 520           check_connection(client, "b.example.test")
 521           check_connection(client, "c.example.test")
 522
 523       with subtest("Certificates and accounts have safe + valid permissions"):
 524           # Nginx will set the group appropriately when enableACME is used
 525           group = "nginx"
 526           webserver.succeed(
 527               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 528           )
 529           webserver.succeed(
 530               f"test $(stat -L -c '%a %U %G' /var/lib/acme/.lego/a.example.test/**/a.example.test* | tee /dev/stderr | grep '600 acme {group}' | wc -l) -eq 4"
 531           )
 532           webserver.succeed(
 533               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test | tee /dev/stderr | grep '750 acme {group}' | wc -l) -eq 1"
 534           )
 535           webserver.succeed(
 536               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
 537           )
 538
 539       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
 540       with subtest("Can generate valid selfsigned certs"):
 541           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
 542           webserver.succeed("systemctl start acme-selfsigned-a.example.test.service")
 543           check_fullchain(webserver, "a.example.test")
 544           check_issuer(webserver, "a.example.test", "minica")
 545           # Check selfsigned permissions
 546           webserver.succeed(
 547               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 548           )
 549           # Will succeed if nginx can load the certs
 550           webserver.succeed("systemctl start nginx-config-reload.service")
 551
 552       with subtest("Correctly implements OCSP stapling"):
 553           switch_to(webserver, "ocsp-stapling")
 554           webserver.wait_for_unit("acme-finished-a.example.test.target")
 555           check_stapling(client, "a.example.test")
 556
 557       with subtest("Can request certificate with HTTP-01 using lego's internal web server"):
 558           switch_to(webserver, "lego-server")
 559           webserver.wait_for_unit("acme-finished-lego.example.test.target")
 560           webserver.wait_for_unit("nginx.service")
 561           webserver.succeed("echo HENLO && systemctl cat nginx.service")
 562           webserver.succeed("test \"$(stat -c '%U' /var/lib/acme/* | uniq)\" = \"root\"")
 563           check_connection(client, "a.example.test")
 564           check_connection(client, "lego.example.test")
 565
 566       with subtest("Can request certificate with HTTP-01 when nginx startup is delayed"):
 567           webserver.execute("systemctl stop nginx")
 568           switch_to(webserver, "slow-startup")
 569           webserver.wait_for_unit("acme-finished-slow.example.test.target")
 570           check_issuer(webserver, "slow.example.test", "pebble")
 571           webserver.wait_for_unit("nginx.service")
 572           check_connection(client, "slow.example.test")
 573
 574       with subtest("Works with caddy"):
 575           switch_to(webserver, "caddy")
 576           webserver.wait_for_unit("acme-finished-example.test.target")
 577           webserver.wait_for_unit("caddy.service")
 578           # FIXME reloading caddy is not sufficient to load new certs.
 579           # Restart it manually until this is fixed.
 580           webserver.succeed("systemctl restart caddy.service")
 581           check_connection(client, "a.example.test")
 582
 583       with subtest("security.acme changes reflect on caddy"):
 584           switch_to(webserver, "caddy-change-acme-conf")
 585           webserver.wait_for_unit("acme-finished-example.test.target")
 586           webserver.wait_for_unit("caddy.service")
 587           # FIXME reloading caddy is not sufficient to load new certs.
 588           # Restart it manually until this is fixed.
 589           webserver.succeed("systemctl restart caddy.service")
 590           check_connection_key_bits(client, "a.example.test", "384")
 591
 592       domains = ["http", "dns", "wildcard"]
 593       for server, logsrc in [
 594           ("nginx", "journalctl -n 30 -u nginx.service"),
 595           ("httpd", "tail -n 30 /var/log/httpd/*.log"),
 596       ]:
 597           wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
 598           with subtest(f"Works with {server}"):
 599               try:
 600                   switch_to(webserver, server)
 601                   # Skip wildcard domain for this check ([:-1])
 602                   for domain in domains[:-1]:
 603                       webserver.wait_for_unit(
 604                           f"acme-finished-{server}-{domain}.example.test.target"
 605                       )
 606               except Exception as err:
 607                   _, output = webserver.execute(
 608                       f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
 609                   )
 610                   print(output)
 611                   raise err
 612
 613               wait_for_server()
 614
 615               for domain in domains[:-1]:
 616                   check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
 617               for domain in domains:
 618                   check_connection(client, f"{server}-{domain}.example.test")
 619                   check_connection(client, f"{server}-{domain}-alias.example.test")
 620
 621           test_domain = f"{server}-{domains[0]}.example.test"
 622
 623           with subtest(f"Can reload {server} when timer triggers renewal"):
 624               # Switch to selfsigned first
 625               webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
 626               webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
 627               check_issuer(webserver, test_domain, "minica")
 628               webserver.succeed(f"systemctl start {server}-config-reload.service")
 629               webserver.succeed(f"systemctl start test-renew-{server}.target")
 630               check_issuer(webserver, test_domain, "pebble")
 631               check_connection(client, test_domain)
 632
 633           with subtest("Can remove an alias from a domain + cert is updated"):
 634               test_alias = f"{server}-{domains[0]}-alias.example.test"
 635               switch_to(webserver, f"{server}-remove-alias")
 636               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 637               wait_for_server()
 638               check_connection(client, test_domain)
 639               rc, _s = client.execute(
 640                   f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
 641                   " </dev/null 2>/dev/null | openssl x509 -noout -text"
 642                   f" | grep DNS: | grep {test_alias}"
 643               )
 644               assert rc > 0, "Removed extraDomainName was not removed from the cert"
 645
 646           with subtest("security.acme changes reflect on web server"):
 647               # Switch back to normal server config first, reset everything.
 648               switch_to(webserver, server)
 649               wait_for_server()
 650               switch_to(webserver, f"{server}-change-acme-conf")
 651               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 652               wait_for_server()
 653               check_connection_key_bits(client, test_domain, "384")
 654     '';
 655 }