nixos/tests/acme.nix

   1 { config, lib, ... }: let
   2
   3   pkgs = config.node.pkgs;
   4
   5   commonConfig = ./common/acme/client;
   6
   7   dnsServerIP = nodes: nodes.dnsserver.networking.primaryIPAddress;
   8
   9   dnsScript = nodes: let
  10     dnsAddress = dnsServerIP nodes;
  11   in pkgs.writeShellScript "dns-hook.sh" ''
  12     set -euo pipefail
  13     echo '[INFO]' "[$2]" 'dns-hook.sh' $*
  14     if [ "$1" = "present" ]; then
  15       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'", "value": "'"$3"'"}' http://${dnsAddress}:8055/set-txt
  16     else
  17       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'"}' http://${dnsAddress}:8055/clear-txt
  18     fi
  19   '';
  20
  21   dnsConfig = nodes: {
  22     dnsProvider = "exec";
  23     dnsPropagationCheck = false;
  24     environmentFile = pkgs.writeText "wildcard.env" ''
  25       EXEC_PATH=${dnsScript nodes}
  26       EXEC_POLLING_INTERVAL=1
  27       EXEC_PROPAGATION_TIMEOUT=1
  28       EXEC_SEQUENCE_INTERVAL=1
  29     '';
  30   };
  31
  32   documentRoot = pkgs.runCommand "docroot" {} ''
  33     mkdir -p "$out"
  34     echo hello world > "$out/index.html"
  35   '';
  36
  37   vhostBase = {
  38     forceSSL = true;
  39     locations."/".root = documentRoot;
  40   };
  41
  42   vhostBaseHttpd = {
  43     forceSSL = true;
  44     inherit documentRoot;
  45   };
  46
  47   simpleConfig = {
  48     security.acme = {
  49       certs."http.example.test" = {
  50         listenHTTP = ":80";
  51       };
  52     };
  53
  54     networking.firewall.allowedTCPPorts = [ 80 ];
  55   };
  56
  57   # Base specialisation config for testing general ACME features
  58   webserverBasicConfig = {
  59     services.nginx.enable = true;
  60     services.nginx.virtualHosts."a.example.test" = vhostBase // {
  61       enableACME = true;
  62     };
  63   };
  64
  65   # Generate specialisations for testing a web server
  66   mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
  67     baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
  68       {
  69         security.acme = {
  70           defaults = (dnsConfig nodes);
  71           # One manual wildcard cert
  72           certs."example.test" = {
  73             domain = "*.example.test";
  74           };
  75         };
  76
  77         users.users."${config.services."${server}".user}".extraGroups = ["acme"];
  78
  79         services."${server}" = {
  80           enable = true;
  81           virtualHosts = {
  82             # Run-of-the-mill vhost using HTTP-01 validation
  83             "${server}-http.example.test" = vhostBaseData // {
  84               serverAliases = [ "${server}-http-alias.example.test" ];
  85               enableACME = true;
  86             };
  87
  88             # Another which inherits the DNS-01 config
  89             "${server}-dns.example.test" = vhostBaseData // {
  90               serverAliases = [ "${server}-dns-alias.example.test" ];
  91               enableACME = true;
  92               # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
  93               # webroot + dnsProvider are mutually exclusive.
  94               acmeRoot = null;
  95             };
  96
  97             # One using the wildcard certificate
  98             "${server}-wildcard.example.test" = vhostBaseData // {
  99               serverAliases = [ "${server}-wildcard-alias.example.test" ];
 100               useACMEHost = "example.test";
 101             };
 102           };
 103         };
 104
 105         # Used to determine if service reload was triggered
 106         systemd.targets."test-renew-${server}" = {
 107           wants = [ "acme-${server}-http.example.test.service" ];
 108           after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
 109         };
 110       }
 111       specialConfig
 112       extraConfig
 113     ];
 114   in {
 115     "${server}".configuration = { nodes, config, ... }: baseConfig {
 116       inherit nodes config;
 117     };
 118
 119     # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
 120     "${server}-remove-alias".configuration = { nodes, config, ... }: baseConfig {
 121       inherit nodes config;
 122       specialConfig = {
 123         # Remove an alias, but create a standalone vhost in its place for testing.
 124         # This configuration results in certificate errors as useACMEHost does not imply
 125         # append extraDomains, and thus we can validate the SAN is removed.
 126         services."${server}" = {
 127           virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
 128           virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
 129             useACMEHost = "${server}-http.example.test";
 130           };
 131         };
 132       };
 133     };
 134
 135     # Test that the server reloads when only the acme configuration is changed.
 136     "${server}-change-acme-conf".configuration = { nodes, config, ... }: baseConfig {
 137       inherit nodes config;
 138       specialConfig = {
 139         security.acme.certs."${server}-http.example.test" = {
 140           keyType = "ec384";
 141           # Also test that postRun is exec'd as root
 142           postRun = "id | grep root";
 143         };
 144       };
 145     };
 146   };
 147
 148 in {
 149   name = "acme";
 150   meta = {
 151     maintainers = lib.teams.acme.members;
 152     # Hard timeout in seconds. Average run time is about 7 minutes.
 153     timeout = 1800;
 154   };
 155
 156   nodes = {
 157     # The fake ACME server which will respond to client requests
 158     acme = { nodes, ... }: {
 159       imports = [ ./common/acme/server ];
 160       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 161     };
 162
 163     # A fake DNS server which can be configured with records as desired
 164     # Used to test DNS-01 challenge
 165     dnsserver = { nodes, ... }: {
 166       networking.firewall.allowedTCPPorts = [ 8055 53 ];
 167       networking.firewall.allowedUDPPorts = [ 53 ];
 168       systemd.services.pebble-challtestsrv = {
 169         enable = true;
 170         description = "Pebble ACME challenge test server";
 171         wantedBy = [ "network.target" ];
 172         serviceConfig = {
 173           ExecStart = "${pkgs.pebble}/bin/pebble-challtestsrv -dns01 ':53' -defaultIPv6 '' -defaultIPv4 '${nodes.webserver.networking.primaryIPAddress}'";
 174           # Required to bind on privileged ports.
 175           AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
 176         };
 177       };
 178     };
 179
 180     # A web server which will be the node requesting certs
 181     webserver = { nodes, config, ... }: {
 182       imports = [ commonConfig ];
 183       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 184       networking.firewall.allowedTCPPorts = [ 80 443 ];
 185
 186       # OpenSSL will be used for more thorough certificate validation
 187       environment.systemPackages = [ pkgs.openssl ];
 188
 189       # Set log level to info so that we can see when the service is reloaded
 190       services.nginx.logError = "stderr info";
 191
 192       specialisation = {
 193         # Tests HTTP-01 verification using Lego's built-in web server
 194         http01lego.configuration = simpleConfig;
 195
 196         renew.configuration = lib.mkMerge [
 197           simpleConfig
 198           {
 199             # Pebble provides 5 year long certs,
 200             # needs to be higher than that to test renewal
 201             security.acme.certs."http.example.test".validMinDays = 9999;
 202           }
 203         ];
 204
 205         # Tests that account creds can be safely changed.
 206         accountchange.configuration = lib.mkMerge [
 207           simpleConfig
 208           {
 209             security.acme.certs."http.example.test".email = "admin@example.test";
 210           }
 211         ];
 212
 213         # First derivation used to test general ACME features
 214         general.configuration = { ... }: let
 215           caDomain = nodes.acme.test-support.acme.caDomain;
 216           email = config.security.acme.defaults.email;
 217           # Exit 99 to make it easier to track if this is the reason a renew failed
 218           accountCreateTester = ''
 219             test -e accounts/${caDomain}/${email}/account.json || exit 99
 220           '';
 221         in lib.mkMerge [
 222           webserverBasicConfig
 223           {
 224             # Used to test that account creation is collated into one service.
 225             # These should not run until after acme-finished-a.example.test.target
 226             systemd.services."b.example.test".preStart = accountCreateTester;
 227             systemd.services."c.example.test".preStart = accountCreateTester;
 228
 229             services.nginx.virtualHosts."b.example.test" = vhostBase // {
 230               enableACME = true;
 231             };
 232             services.nginx.virtualHosts."c.example.test" = vhostBase // {
 233               enableACME = true;
 234             };
 235           }
 236         ];
 237
 238         # Test OCSP Stapling
 239         ocsp-stapling.configuration = { ... }: lib.mkMerge [
 240           webserverBasicConfig
 241           {
 242             security.acme.certs."a.example.test".ocspMustStaple = true;
 243             services.nginx.virtualHosts."a.example.test" = {
 244               extraConfig = ''
 245                 ssl_stapling on;
 246                 ssl_stapling_verify on;
 247               '';
 248             };
 249           }
 250         ];
 251
 252         # Validate service relationships by adding a slow start service to nginx' wants.
 253         # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
 254         slow-startup.configuration = { ... }: lib.mkMerge [
 255           webserverBasicConfig
 256           {
 257             systemd.services.my-slow-service = {
 258               wantedBy = [ "multi-user.target" "nginx.service" ];
 259               before = [ "nginx.service" ];
 260               preStart = "sleep 5";
 261               script = "${pkgs.python3}/bin/python -m http.server";
 262             };
 263
 264             services.nginx.virtualHosts."slow.example.test" = {
 265               forceSSL = true;
 266               enableACME = true;
 267               locations."/".proxyPass = "http://localhost:8000";
 268             };
 269           }
 270         ];
 271
 272         concurrency-limit.configuration = {pkgs, ...}: lib.mkMerge [
 273           webserverBasicConfig {
 274             security.acme.maxConcurrentRenewals = 1;
 275
 276             services.nginx.virtualHosts = {
 277               "f.example.test" = vhostBase // {
 278                 enableACME = true;
 279               };
 280               "g.example.test" = vhostBase // {
 281                 enableACME = true;
 282               };
 283               "h.example.test" = vhostBase // {
 284                 enableACME = true;
 285               };
 286             };
 287
 288             systemd.services = {
 289               # check for mutual exclusion of starting renew services
 290               "acme-f.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-f" ''
 291                 test "$(systemctl is-active acme-{g,h}.example.test.service | grep activating | wc -l)" -le 0
 292                 '');
 293               "acme-g.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-g" ''
 294                 test "$(systemctl is-active acme-{f,h}.example.test.service | grep activating | wc -l)" -le 0
 295                 '');
 296               "acme-h.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-h" ''
 297                 test "$(systemctl is-active acme-{g,f}.example.test.service | grep activating | wc -l)" -le 0
 298                 '');
 299               };
 300           }
 301         ];
 302
 303         # Test lego internal server (listenHTTP option)
 304         # Also tests useRoot option
 305         lego-server.configuration = { ... }: {
 306           security.acme.useRoot = true;
 307           security.acme.certs."lego.example.test" = {
 308             listenHTTP = ":80";
 309             group = "nginx";
 310           };
 311           services.nginx.enable = true;
 312           services.nginx.virtualHosts."lego.example.test" = {
 313             useACMEHost = "lego.example.test";
 314             onlySSL = true;
 315           };
 316         };
 317
 318       # Test compatibility with Caddy
 319       # It only supports useACMEHost, hence not using mkServerConfigs
 320       } // (let
 321         baseCaddyConfig = { nodes, config, ... }: {
 322           security.acme = {
 323             defaults = (dnsConfig nodes);
 324             # One manual wildcard cert
 325             certs."example.test" = {
 326               domain = "*.example.test";
 327             };
 328           };
 329
 330           users.users."${config.services.caddy.user}".extraGroups = ["acme"];
 331
 332           services.caddy = {
 333             enable = true;
 334             virtualHosts."a.example.test" = {
 335               useACMEHost = "example.test";
 336               extraConfig = ''
 337                 root * ${documentRoot}
 338               '';
 339             };
 340           };
 341         };
 342       in {
 343         caddy.configuration = baseCaddyConfig;
 344
 345         # Test that the server reloads when only the acme configuration is changed.
 346         "caddy-change-acme-conf".configuration = { nodes, config, ... }: lib.mkMerge [
 347           (baseCaddyConfig {
 348             inherit nodes config;
 349           })
 350           {
 351             security.acme.certs."example.test" = {
 352               keyType = "ec384";
 353             };
 354           }
 355         ];
 356
 357       # Test compatibility with Nginx
 358       }) // (mkServerConfigs {
 359           server = "nginx";
 360           group = "nginx";
 361           vhostBaseData = vhostBase;
 362         })
 363
 364       # Test compatibility with Apache HTTPD
 365         // (mkServerConfigs {
 366           server = "httpd";
 367           group = "wwwrun";
 368           vhostBaseData = vhostBaseHttpd;
 369           extraConfig = {
 370             services.httpd.adminAddr = config.security.acme.defaults.email;
 371           };
 372         });
 373     };
 374
 375     # The client will be used to curl the webserver to validate configuration
 376     client = { nodes, ... }: {
 377       imports = [ commonConfig ];
 378       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 379
 380       # OpenSSL will be used for more thorough certificate validation
 381       environment.systemPackages = [ pkgs.openssl ];
 382     };
 383   };
 384
 385   testScript = { nodes, ... }:
 386     let
 387       caDomain = nodes.acme.test-support.acme.caDomain;
 388       newServerSystem = nodes.webserver.config.system.build.toplevel;
 389       switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test";
 390     in
 391     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
 392     # this is because a oneshot goes from inactive => activating => inactive, and never
 393     # reaches the active state. Targets do not have this issue.
 394     ''
 395       import time
 396
 397
 398       TOTAL_RETRIES = 20
 399
 400
 401       class BackoffTracker(object):
 402           delay = 1
 403           increment = 1
 404
 405           def handle_fail(self, retries, message) -> int:
 406               assert retries < TOTAL_RETRIES, message
 407
 408               print(f"Retrying in {self.delay}s, {retries + 1}/{TOTAL_RETRIES}")
 409               time.sleep(self.delay)
 410
 411               # Only increment after the first try
 412               if retries == 0:
 413                   self.delay += self.increment
 414                   self.increment *= 2
 415
 416               return retries + 1
 417
 418
 419       backoff = BackoffTracker()
 420
 421
 422       def switch_to(node, name):
 423           # On first switch, this will create a symlink to the current system so that we can
 424           # quickly switch between derivations
 425           root_specs = "/tmp/specialisation"
 426           node.execute(
 427             f"test -e {root_specs}"
 428             f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
 429           )
 430
 431           switcher_path = f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
 432           rc, _ = node.execute(f"test -e '{switcher_path}'")
 433           if rc > 0:
 434               switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 435
 436           node.succeed(
 437               f"{switcher_path} test"
 438           )
 439
 440
 441       # Ensures the issuer of our cert matches the chain
 442       # and matches the issuer we expect it to be.
 443       # It's a good validation to ensure the cert.pem and fullchain.pem
 444       # are not still selfsigned after verification
 445       def check_issuer(node, cert_name, issuer):
 446           for fname in ("cert.pem", "fullchain.pem"):
 447               actual_issuer = node.succeed(
 448                   f"openssl x509 -noout -issuer -in /var/lib/acme/{cert_name}/{fname}"
 449               ).partition("=")[2]
 450               print(f"{fname} issuer: {actual_issuer}")
 451               assert issuer.lower() in actual_issuer.lower()
 452
 453
 454       # Ensure cert comes before chain in fullchain.pem
 455       def check_fullchain(node, cert_name):
 456           subject_data = node.succeed(
 457               f"openssl crl2pkcs7 -nocrl -certfile /var/lib/acme/{cert_name}/fullchain.pem"
 458               " | openssl pkcs7 -print_certs -noout"
 459           )
 460           for line in subject_data.lower().split("\n"):
 461               if "subject" in line:
 462                   print(f"First subject in fullchain.pem: {line}")
 463                   assert cert_name.lower() in line
 464                   return
 465
 466           assert False
 467
 468
 469       def check_connection(node, domain, retries=0):
 470           result = node.succeed(
 471               "openssl s_client -brief -verify 2 -CAfile /tmp/ca.crt"
 472               f" -servername {domain} -connect {domain}:443 < /dev/null 2>&1"
 473           )
 474
 475           for line in result.lower().split("\n"):
 476               if "verification" in line and "error" in line:
 477                   retries = backoff.handle_fail(retries, f"Failed to connect to https://{domain}")
 478                   return check_connection(node, domain, retries)
 479
 480
 481       def check_connection_key_bits(node, domain, bits, retries=0):
 482           result = node.succeed(
 483               "openssl s_client -CAfile /tmp/ca.crt"
 484               f" -servername {domain} -connect {domain}:443 < /dev/null"
 485               " | openssl x509 -noout -text | grep -i Public-Key"
 486           )
 487           print("Key type:", result)
 488
 489           if bits not in result:
 490               retries = backoff.handle_fail(retries, f"Did not find expected number of bits ({bits}) in key")
 491               return check_connection_key_bits(node, domain, bits, retries)
 492
 493
 494       def check_stapling(node, domain, retries=0):
 495           # Pebble doesn't provide a full OCSP responder, so just check the URL
 496           result = node.succeed(
 497               "openssl s_client -CAfile /tmp/ca.crt"
 498               f" -servername {domain} -connect {domain}:443 < /dev/null"
 499               " | openssl x509 -noout -ocsp_uri"
 500           )
 501           print("OCSP Responder URL:", result)
 502
 503           if "${caDomain}:4002" not in result.lower():
 504               retries = backoff.handle_fail(retries, "OCSP Stapling check failed")
 505               return check_stapling(node, domain, retries)
 506
 507
 508       def download_ca_certs(node, retries=0):
 509           exit_code, _ = node.execute("curl https://${caDomain}:15000/roots/0 > /tmp/ca.crt")
 510           exit_code_2, _ = node.execute(
 511               "curl https://${caDomain}:15000/intermediate-keys/0 >> /tmp/ca.crt"
 512           )
 513
 514           if exit_code + exit_code_2 > 0:
 515               retries = backoff.handle_fail(retries, "Failed to connect to pebble to download root CA certs")
 516               return download_ca_certs(node, retries)
 517
 518
 519       start_all()
 520
 521       dnsserver.wait_for_unit("pebble-challtestsrv.service")
 522       client.wait_for_unit("default.target")
 523
 524       client.succeed(
 525           'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
 526       )
 527
 528       acme.systemctl("start network-online.target")
 529       acme.wait_for_unit("network-online.target")
 530       acme.wait_for_unit("pebble.service")
 531
 532       download_ca_certs(client)
 533
 534       # Perform http-01 w/ lego test first
 535       with subtest("Can request certificate with Lego's built in web server"):
 536           switch_to(webserver, "http01lego")
 537           webserver.wait_for_unit("acme-finished-http.example.test.target")
 538           check_fullchain(webserver, "http.example.test")
 539           check_issuer(webserver, "http.example.test", "pebble")
 540
 541       # Perform renewal test
 542       with subtest("Can renew certificates when they expire"):
 543           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 544           switch_to(webserver, "renew")
 545           webserver.wait_for_unit("acme-finished-http.example.test.target")
 546           check_fullchain(webserver, "http.example.test")
 547           check_issuer(webserver, "http.example.test", "pebble")
 548           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 549           assert hash != hash_after
 550
 551       # Perform account change test
 552       with subtest("Handles email change correctly"):
 553           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 554           switch_to(webserver, "accountchange")
 555           webserver.wait_for_unit("acme-finished-http.example.test.target")
 556           check_fullchain(webserver, "http.example.test")
 557           check_issuer(webserver, "http.example.test", "pebble")
 558           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 559           # Has to do a full run to register account, which creates new certs.
 560           assert hash != hash_after
 561
 562       # Perform general tests
 563       switch_to(webserver, "general")
 564
 565       with subtest("Can request certificate with HTTP-01 challenge"):
 566           webserver.wait_for_unit("acme-finished-a.example.test.target")
 567           check_fullchain(webserver, "a.example.test")
 568           check_issuer(webserver, "a.example.test", "pebble")
 569           webserver.wait_for_unit("nginx.service")
 570           check_connection(client, "a.example.test")
 571
 572       with subtest("Runs 1 cert for account creation before others"):
 573           webserver.wait_for_unit("acme-finished-b.example.test.target")
 574           webserver.wait_for_unit("acme-finished-c.example.test.target")
 575           check_connection(client, "b.example.test")
 576           check_connection(client, "c.example.test")
 577
 578       with subtest("Certificates and accounts have safe + valid permissions"):
 579           # Nginx will set the group appropriately when enableACME is used
 580           group = "nginx"
 581           webserver.succeed(
 582               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 583           )
 584           webserver.succeed(
 585               f"test $(stat -L -c '%a %U %G' /var/lib/acme/.lego/a.example.test/**/a.example.test* | tee /dev/stderr | grep '600 acme {group}' | wc -l) -eq 4"
 586           )
 587           webserver.succeed(
 588               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test | tee /dev/stderr | grep '750 acme {group}' | wc -l) -eq 1"
 589           )
 590           webserver.succeed(
 591               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
 592           )
 593
 594       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
 595       with subtest("Can generate valid selfsigned certs"):
 596           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
 597           webserver.succeed("systemctl start acme-selfsigned-a.example.test.service")
 598           check_fullchain(webserver, "a.example.test")
 599           check_issuer(webserver, "a.example.test", "minica")
 600           # Check selfsigned permissions
 601           webserver.succeed(
 602               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 603           )
 604           # Will succeed if nginx can load the certs
 605           webserver.succeed("systemctl start nginx-config-reload.service")
 606
 607       with subtest("Correctly implements OCSP stapling"):
 608           switch_to(webserver, "ocsp-stapling")
 609           webserver.wait_for_unit("acme-finished-a.example.test.target")
 610           check_stapling(client, "a.example.test")
 611
 612       with subtest("Can request certificate with HTTP-01 using lego's internal web server"):
 613           switch_to(webserver, "lego-server")
 614           webserver.wait_for_unit("acme-finished-lego.example.test.target")
 615           webserver.wait_for_unit("nginx.service")
 616           webserver.succeed("echo HENLO && systemctl cat nginx.service")
 617           webserver.succeed("test \"$(stat -c '%U' /var/lib/acme/* | uniq)\" = \"root\"")
 618           check_connection(client, "a.example.test")
 619           check_connection(client, "lego.example.test")
 620
 621       with subtest("Can request certificate with HTTP-01 when nginx startup is delayed"):
 622           webserver.execute("systemctl stop nginx")
 623           switch_to(webserver, "slow-startup")
 624           webserver.wait_for_unit("acme-finished-slow.example.test.target")
 625           check_issuer(webserver, "slow.example.test", "pebble")
 626           webserver.wait_for_unit("nginx.service")
 627           check_connection(client, "slow.example.test")
 628
 629       with subtest("Can limit concurrency of running renewals"):
 630           switch_to(webserver, "concurrency-limit")
 631           webserver.wait_for_unit("acme-finished-f.example.test.target")
 632           webserver.wait_for_unit("acme-finished-g.example.test.target")
 633           webserver.wait_for_unit("acme-finished-h.example.test.target")
 634           check_connection(client, "f.example.test")
 635           check_connection(client, "g.example.test")
 636           check_connection(client, "h.example.test")
 637
 638       with subtest("Works with caddy"):
 639           switch_to(webserver, "caddy")
 640           webserver.wait_for_unit("acme-finished-example.test.target")
 641           webserver.wait_for_unit("caddy.service")
 642           # FIXME reloading caddy is not sufficient to load new certs.
 643           # Restart it manually until this is fixed.
 644           webserver.succeed("systemctl restart caddy.service")
 645           check_connection(client, "a.example.test")
 646
 647       with subtest("security.acme changes reflect on caddy"):
 648           switch_to(webserver, "caddy-change-acme-conf")
 649           webserver.wait_for_unit("acme-finished-example.test.target")
 650           webserver.wait_for_unit("caddy.service")
 651           # FIXME reloading caddy is not sufficient to load new certs.
 652           # Restart it manually until this is fixed.
 653           webserver.succeed("systemctl restart caddy.service")
 654           check_connection_key_bits(client, "a.example.test", "384")
 655
 656       domains = ["http", "dns", "wildcard"]
 657       for server, logsrc in [
 658           ("nginx", "journalctl -n 30 -u nginx.service"),
 659           ("httpd", "tail -n 30 /var/log/httpd/*.log"),
 660       ]:
 661           wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
 662           with subtest(f"Works with {server}"):
 663               try:
 664                   switch_to(webserver, server)
 665                   # Skip wildcard domain for this check ([:-1])
 666                   for domain in domains[:-1]:
 667                       webserver.wait_for_unit(
 668                           f"acme-finished-{server}-{domain}.example.test.target"
 669                       )
 670               except Exception as err:
 671                   _, output = webserver.execute(
 672                       f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
 673                   )
 674                   print(output)
 675                   raise err
 676
 677               wait_for_server()
 678
 679               for domain in domains[:-1]:
 680                   check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
 681               for domain in domains:
 682                   check_connection(client, f"{server}-{domain}.example.test")
 683                   check_connection(client, f"{server}-{domain}-alias.example.test")
 684
 685           test_domain = f"{server}-{domains[0]}.example.test"
 686
 687           with subtest(f"Can reload {server} when timer triggers renewal"):
 688               # Switch to selfsigned first
 689               webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
 690               webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
 691               check_issuer(webserver, test_domain, "minica")
 692               webserver.succeed(f"systemctl start {server}-config-reload.service")
 693               webserver.succeed(f"systemctl start test-renew-{server}.target")
 694               check_issuer(webserver, test_domain, "pebble")
 695               check_connection(client, test_domain)
 696
 697           with subtest("Can remove an alias from a domain + cert is updated"):
 698               test_alias = f"{server}-{domains[0]}-alias.example.test"
 699               switch_to(webserver, f"{server}-remove-alias")
 700               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 701               wait_for_server()
 702               check_connection(client, test_domain)
 703               rc, _s = client.execute(
 704                   f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
 705                   " </dev/null 2>/dev/null | openssl x509 -noout -text"
 706                   f" | grep DNS: | grep {test_alias}"
 707               )
 708               assert rc > 0, "Removed extraDomainName was not removed from the cert"
 709
 710           with subtest("security.acme changes reflect on web server"):
 711               # Switch back to normal server config first, reset everything.
 712               switch_to(webserver, server)
 713               wait_for_server()
 714               switch_to(webserver, f"{server}-change-acme-conf")
 715               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 716               wait_for_server()
 717               check_connection_key_bits(client, test_domain, "384")
 718     '';
 719 }