nixos/tests/acme.nix

   1 { config, lib, ... }: let
   2
   3   pkgs = config.node.pkgs;
   4
   5   commonConfig = ./common/acme/client;
   6
   7   dnsServerIP = nodes: nodes.dnsserver.networking.primaryIPAddress;
   8
   9   dnsScript = nodes: let
  10     dnsAddress = dnsServerIP nodes;
  11   in pkgs.writeShellScript "dns-hook.sh" ''
  12     set -euo pipefail
  13     echo '[INFO]' "[$2]" 'dns-hook.sh' $*
  14     if [ "$1" = "present" ]; then
  15       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'", "value": "'"$3"'"}' http://${dnsAddress}:8055/set-txt
  16     else
  17       ${pkgs.curl}/bin/curl --data '{"host": "'"$2"'"}' http://${dnsAddress}:8055/clear-txt
  18     fi
  19   '';
  20
  21   dnsConfig = nodes: {
  22     dnsProvider = "exec";
  23     dnsPropagationCheck = false;
  24     environmentFile = pkgs.writeText "wildcard.env" ''
  25       EXEC_PATH=${dnsScript nodes}
  26       EXEC_POLLING_INTERVAL=1
  27       EXEC_PROPAGATION_TIMEOUT=1
  28       EXEC_SEQUENCE_INTERVAL=1
  29     '';
  30   };
  31
  32   documentRoot = pkgs.runCommand "docroot" {} ''
  33     mkdir -p "$out"
  34     echo hello world > "$out/index.html"
  35   '';
  36
  37   vhostBase = {
  38     forceSSL = true;
  39     locations."/".root = documentRoot;
  40   };
  41
  42   vhostBaseHttpd = {
  43     forceSSL = true;
  44     inherit documentRoot;
  45   };
  46
  47   simpleConfig = {
  48     security.acme = {
  49       certs."http.example.test" = {
  50         listenHTTP = ":80";
  51       };
  52     };
  53
  54     networking.firewall.allowedTCPPorts = [ 80 ];
  55   };
  56
  57   # Base specialisation config for testing general ACME features
  58   webserverBasicConfig = {
  59     services.nginx.enable = true;
  60     services.nginx.virtualHosts."a.example.test" = vhostBase // {
  61       enableACME = true;
  62     };
  63   };
  64
  65   # Generate specialisations for testing a web server
  66   mkServerConfigs = { server, group, vhostBaseData, extraConfig ? {} }: let
  67     baseConfig = { nodes, config, specialConfig ? {} }: lib.mkMerge [
  68       {
  69         security.acme = {
  70           defaults = (dnsConfig nodes);
  71           # One manual wildcard cert
  72           certs."example.test" = {
  73             domain = "*.example.test";
  74           };
  75         };
  76
  77         users.users."${config.services."${server}".user}".extraGroups = ["acme"];
  78
  79         services."${server}" = {
  80           enable = true;
  81           virtualHosts = {
  82             # Run-of-the-mill vhost using HTTP-01 validation
  83             "${server}-http.example.test" = vhostBaseData // {
  84               serverAliases = [ "${server}-http-alias.example.test" ];
  85               enableACME = true;
  86             };
  87
  88             # Another which inherits the DNS-01 config
  89             "${server}-dns.example.test" = vhostBaseData // {
  90               serverAliases = [ "${server}-dns-alias.example.test" ];
  91               enableACME = true;
  92               # Set acmeRoot to null instead of using the default of "/var/lib/acme/acme-challenge"
  93               # webroot + dnsProvider are mutually exclusive.
  94               acmeRoot = null;
  95             };
  96
  97             # One using the wildcard certificate
  98             "${server}-wildcard.example.test" = vhostBaseData // {
  99               serverAliases = [ "${server}-wildcard-alias.example.test" ];
 100               useACMEHost = "example.test";
 101             };
 102           } // (lib.optionalAttrs (server == "nginx") {
 103             # The nginx module supports using a different key than the hostname
 104             different-key = vhostBaseData // {
 105               serverName = "${server}-different-key.example.test";
 106               serverAliases = [ "${server}-different-key-alias.example.test" ];
 107               enableACME = true;
 108             };
 109           });
 110         };
 111
 112         # Used to determine if service reload was triggered
 113         systemd.targets."test-renew-${server}" = {
 114           wants = [ "acme-${server}-http.example.test.service" ];
 115           after = [ "acme-${server}-http.example.test.service" "${server}-config-reload.service" ];
 116         };
 117       }
 118       specialConfig
 119       extraConfig
 120     ];
 121   in {
 122     "${server}".configuration = { nodes, config, ... }: baseConfig {
 123       inherit nodes config;
 124     };
 125
 126     # Test that server reloads when an alias is removed (and subsequently test removal works in acme)
 127     "${server}_remove_alias".configuration = { nodes, config, ... }: baseConfig {
 128       inherit nodes config;
 129       specialConfig = {
 130         # Remove an alias, but create a standalone vhost in its place for testing.
 131         # This configuration results in certificate errors as useACMEHost does not imply
 132         # append extraDomains, and thus we can validate the SAN is removed.
 133         services."${server}" = {
 134           virtualHosts."${server}-http.example.test".serverAliases = lib.mkForce [];
 135           virtualHosts."${server}-http-alias.example.test" = vhostBaseData // {
 136             useACMEHost = "${server}-http.example.test";
 137           };
 138         };
 139       };
 140     };
 141
 142     # Test that the server reloads when only the acme configuration is changed.
 143     "${server}_change_acme_conf".configuration = { nodes, config, ... }: baseConfig {
 144       inherit nodes config;
 145       specialConfig = {
 146         security.acme.certs."${server}-http.example.test" = {
 147           keyType = "ec384";
 148           # Also test that postRun is exec'd as root
 149           postRun = "id | grep root";
 150         };
 151       };
 152     };
 153   };
 154
 155 in {
 156   name = "acme";
 157   meta = {
 158     maintainers = lib.teams.acme.members;
 159     # Hard timeout in seconds. Average run time is about 7 minutes.
 160     timeout = 1800;
 161   };
 162
 163   nodes = {
 164     # The fake ACME server which will respond to client requests
 165     acme = { nodes, ... }: {
 166       imports = [ ./common/acme/server ];
 167       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 168     };
 169
 170     # A fake DNS server which can be configured with records as desired
 171     # Used to test DNS-01 challenge
 172     dnsserver = { nodes, ... }: {
 173       networking.firewall.allowedTCPPorts = [ 8055 53 ];
 174       networking.firewall.allowedUDPPorts = [ 53 ];
 175       systemd.services.pebble-challtestsrv = {
 176         enable = true;
 177         description = "Pebble ACME challenge test server";
 178         wantedBy = [ "network.target" ];
 179         serviceConfig = {
 180           ExecStart = "${pkgs.pebble}/bin/pebble-challtestsrv -dns01 ':53' -defaultIPv6 '' -defaultIPv4 '${nodes.webserver.networking.primaryIPAddress}'";
 181           # Required to bind on privileged ports.
 182           AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
 183         };
 184       };
 185     };
 186
 187     # A web server which will be the node requesting certs
 188     webserver = { nodes, config, ... }: {
 189       imports = [ commonConfig ];
 190       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 191       networking.firewall.allowedTCPPorts = [ 80 443 ];
 192
 193       # OpenSSL will be used for more thorough certificate validation
 194       environment.systemPackages = [ pkgs.openssl ];
 195
 196       # Set log level to info so that we can see when the service is reloaded
 197       services.nginx.logError = "stderr info";
 198
 199       specialisation = {
 200         # Tests HTTP-01 verification using Lego's built-in web server
 201         http01lego.configuration = simpleConfig;
 202
 203         # account hash generation with default server from <= 23.11
 204         http01lego_legacyAccountHash.configuration = lib.mkMerge [
 205           simpleConfig
 206           {
 207             security.acme.defaults.server = lib.mkForce null;
 208           }
 209         ];
 210
 211         renew.configuration = lib.mkMerge [
 212           simpleConfig
 213           {
 214             # Pebble provides 5 year long certs,
 215             # needs to be higher than that to test renewal
 216             security.acme.certs."http.example.test".validMinDays = 9999;
 217           }
 218         ];
 219
 220         # Tests that account creds can be safely changed.
 221         accountchange.configuration = lib.mkMerge [
 222           simpleConfig
 223           {
 224             security.acme.certs."http.example.test".email = "admin@example.test";
 225           }
 226         ];
 227
 228         # First derivation used to test general ACME features
 229         general.configuration = { ... }: let
 230           caDomain = nodes.acme.test-support.acme.caDomain;
 231           email = config.security.acme.defaults.email;
 232           # Exit 99 to make it easier to track if this is the reason a renew failed
 233           accountCreateTester = ''
 234             test -e accounts/${caDomain}/${email}/account.json || exit 99
 235           '';
 236         in lib.mkMerge [
 237           webserverBasicConfig
 238           {
 239             # Used to test that account creation is collated into one service.
 240             # These should not run until after acme-finished-a.example.test.target
 241             systemd.services."b.example.test".preStart = accountCreateTester;
 242             systemd.services."c.example.test".preStart = accountCreateTester;
 243
 244             services.nginx.virtualHosts."b.example.test" = vhostBase // {
 245               enableACME = true;
 246             };
 247             services.nginx.virtualHosts."c.example.test" = vhostBase // {
 248               enableACME = true;
 249             };
 250           }
 251         ];
 252
 253         # Test OCSP Stapling
 254         ocsp_stapling.configuration = { ... }: lib.mkMerge [
 255           webserverBasicConfig
 256           {
 257             security.acme.certs."a.example.test".ocspMustStaple = true;
 258             services.nginx.virtualHosts."a.example.test" = {
 259               extraConfig = ''
 260                 ssl_stapling on;
 261                 ssl_stapling_verify on;
 262               '';
 263             };
 264           }
 265         ];
 266
 267         # Validate service relationships by adding a slow start service to nginx' wants.
 268         # Reproducer for https://github.com/NixOS/nixpkgs/issues/81842
 269         slow_startup.configuration = { ... }: lib.mkMerge [
 270           webserverBasicConfig
 271           {
 272             systemd.services.my-slow-service = {
 273               wantedBy = [ "multi-user.target" "nginx.service" ];
 274               before = [ "nginx.service" ];
 275               preStart = "sleep 5";
 276               script = "${pkgs.python3}/bin/python -m http.server";
 277             };
 278
 279             services.nginx.virtualHosts."slow.example.test" = {
 280               forceSSL = true;
 281               enableACME = true;
 282               locations."/".proxyPass = "http://localhost:8000";
 283             };
 284           }
 285         ];
 286
 287         concurrency_limit.configuration = {pkgs, ...}: lib.mkMerge [
 288           webserverBasicConfig {
 289             security.acme.maxConcurrentRenewals = 1;
 290
 291             services.nginx.virtualHosts = {
 292               "f.example.test" = vhostBase // {
 293                 enableACME = true;
 294               };
 295               "g.example.test" = vhostBase // {
 296                 enableACME = true;
 297               };
 298               "h.example.test" = vhostBase // {
 299                 enableACME = true;
 300               };
 301             };
 302
 303             systemd.services = {
 304               # check for mutual exclusion of starting renew services
 305               "acme-f.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-f" ''
 306                 test "$(systemctl is-active acme-{g,h}.example.test.service | grep activating | wc -l)" -le 0
 307                 '');
 308               "acme-g.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-g" ''
 309                 test "$(systemctl is-active acme-{f,h}.example.test.service | grep activating | wc -l)" -le 0
 310                 '');
 311               "acme-h.example.test".serviceConfig.ExecPreStart = "+" + (pkgs.writeShellScript "test-h" ''
 312                 test "$(systemctl is-active acme-{g,f}.example.test.service | grep activating | wc -l)" -le 0
 313                 '');
 314               };
 315           }
 316         ];
 317
 318         # Test lego internal server (listenHTTP option)
 319         # Also tests useRoot option
 320         lego_server.configuration = { ... }: {
 321           security.acme.useRoot = true;
 322           security.acme.certs."lego.example.test" = {
 323             listenHTTP = ":80";
 324             group = "nginx";
 325           };
 326           services.nginx.enable = true;
 327           services.nginx.virtualHosts."lego.example.test" = {
 328             useACMEHost = "lego.example.test";
 329             onlySSL = true;
 330           };
 331         };
 332
 333       # Test compatibility with Caddy
 334       # It only supports useACMEHost, hence not using mkServerConfigs
 335       } // (let
 336         baseCaddyConfig = { nodes, config, ... }: {
 337           security.acme = {
 338             defaults = (dnsConfig nodes);
 339             # One manual wildcard cert
 340             certs."example.test" = {
 341               domain = "*.example.test";
 342             };
 343           };
 344
 345           users.users."${config.services.caddy.user}".extraGroups = ["acme"];
 346
 347           services.caddy = {
 348             enable = true;
 349             virtualHosts."a.example.test" = {
 350               useACMEHost = "example.test";
 351               extraConfig = ''
 352                 root * ${documentRoot}
 353               '';
 354             };
 355           };
 356         };
 357       in {
 358         caddy.configuration = baseCaddyConfig;
 359
 360         # Test that the server reloads when only the acme configuration is changed.
 361         "caddy_change_acme_conf".configuration = { nodes, config, ... }: lib.mkMerge [
 362           (baseCaddyConfig {
 363             inherit nodes config;
 364           })
 365           {
 366             security.acme.certs."example.test" = {
 367               keyType = "ec384";
 368             };
 369           }
 370         ];
 371
 372       # Test compatibility with Nginx
 373       }) // (mkServerConfigs {
 374           server = "nginx";
 375           group = "nginx";
 376           vhostBaseData = vhostBase;
 377         })
 378
 379       # Test compatibility with Apache HTTPD
 380         // (mkServerConfigs {
 381           server = "httpd";
 382           group = "wwwrun";
 383           vhostBaseData = vhostBaseHttpd;
 384           extraConfig = {
 385             services.httpd.adminAddr = config.security.acme.defaults.email;
 386           };
 387         });
 388     };
 389
 390     # The client will be used to curl the webserver to validate configuration
 391     client = { nodes, ... }: {
 392       imports = [ commonConfig ];
 393       networking.nameservers = lib.mkForce [ (dnsServerIP nodes) ];
 394
 395       # OpenSSL will be used for more thorough certificate validation
 396       environment.systemPackages = [ pkgs.openssl ];
 397     };
 398   };
 399
 400   testScript = { nodes, ... }:
 401     let
 402       caDomain = nodes.acme.test-support.acme.caDomain;
 403     in
 404     # Note, wait_for_unit does not work for oneshot services that do not have RemainAfterExit=true,
 405     # this is because a oneshot goes from inactive => activating => inactive, and never
 406     # reaches the active state. Targets do not have this issue.
 407     ''
 408       import time
 409
 410       TOTAL_RETRIES = 20
 411
 412
 413       class BackoffTracker(object):
 414           delay = 1
 415           increment = 1
 416
 417           def handle_fail(self, retries, message) -> int:
 418               assert retries < TOTAL_RETRIES, message
 419
 420               print(f"Retrying in {self.delay}s, {retries + 1}/{TOTAL_RETRIES}")
 421               time.sleep(self.delay)
 422
 423               # Only increment after the first try
 424               if retries == 0:
 425                   self.delay += self.increment
 426                   self.increment *= 2
 427
 428               return retries + 1
 429
 430           def protect(self, func):
 431               def wrapper(*args, retries: int = 0, **kwargs):
 432                   try:
 433                       return func(*args, **kwargs)
 434                   except Exception as err:
 435                       retries = self.handle_fail(retries, err.args)
 436                       return wrapper(*args, retries=retries, **kwargs)
 437
 438               return wrapper
 439
 440
 441       backoff = BackoffTracker()
 442
 443
 444       def switch_to(node, name, allow_fail=False):
 445           # On first switch, this will create a symlink to the current system so that we can
 446           # quickly switch between derivations
 447           root_specs = "/tmp/specialisation"
 448           node.execute(
 449               f"test -e {root_specs}"
 450               f" || ln -s $(readlink /run/current-system)/specialisation {root_specs}"
 451           )
 452
 453           switcher_path = (
 454               f"/run/current-system/specialisation/{name}/bin/switch-to-configuration"
 455           )
 456           rc, _ = node.execute(f"test -e '{switcher_path}'")
 457           if rc > 0:
 458               switcher_path = f"/tmp/specialisation/{name}/bin/switch-to-configuration"
 459
 460           if not allow_fail:
 461             node.succeed(
 462                 f"{switcher_path} test"
 463             )
 464           else:
 465             node.execute(
 466                 f"{switcher_path} test"
 467             )
 468
 469
 470       # Ensures the issuer of our cert matches the chain
 471       # and matches the issuer we expect it to be.
 472       # It's a good validation to ensure the cert.pem and fullchain.pem
 473       # are not still selfsigned after verification
 474       def check_issuer(node, cert_name, issuer):
 475           for fname in ("cert.pem", "fullchain.pem"):
 476               actual_issuer = node.succeed(
 477                   f"openssl x509 -noout -issuer -in /var/lib/acme/{cert_name}/{fname}"
 478               ).partition("=")[2]
 479               assert (
 480                   issuer.lower() in actual_issuer.lower()
 481               ), f"{fname} issuer mismatch. Expected {issuer} got {actual_issuer}"
 482
 483
 484       # Ensure cert comes before chain in fullchain.pem
 485       def check_fullchain(node, cert_name):
 486           cert_file = f"/var/lib/acme/{cert_name}/fullchain.pem"
 487           num_certs = node.succeed(f"grep -o 'END CERTIFICATE' {cert_file}")
 488           assert len(num_certs.strip().split("\n")) > 1, "Insufficient certs in fullchain.pem"
 489
 490           first_cert_data = node.succeed(
 491               f"grep -m1 -B50 'END CERTIFICATE' {cert_file}"
 492               " | openssl x509 -noout -text"
 493           )
 494           for line in first_cert_data.lower().split("\n"):
 495               if "dns:" in line:
 496                   print(f"First DNSName in fullchain.pem: {line}")
 497                   assert cert_name.lower() in line, f"{cert_name} not found in {line}"
 498                   return
 499
 500           assert False
 501
 502
 503       @backoff.protect
 504       def check_connection(node, domain):
 505           result = node.succeed(
 506               "openssl s_client -brief -verify 2 -CAfile /tmp/ca.crt"
 507               f" -servername {domain} -connect {domain}:443 < /dev/null 2>&1"
 508           )
 509
 510           for line in result.lower().split("\n"):
 511               assert not (
 512                   "verification" in line and "error" in line
 513               ), f"Failed to connect to https://{domain}"
 514
 515
 516       @backoff.protect
 517       def check_connection_key_bits(node, domain, bits):
 518           result = node.succeed(
 519               "openssl s_client -CAfile /tmp/ca.crt"
 520               f" -servername {domain} -connect {domain}:443 < /dev/null"
 521               " | openssl x509 -noout -text | grep -i Public-Key"
 522           )
 523           print("Key type:", result)
 524
 525           assert bits in result, f"Did not find expected number of bits ({bits}) in key"
 526
 527
 528       @backoff.protect
 529       def check_stapling(node, domain):
 530           # Pebble doesn't provide a full OCSP responder, so just check the URL
 531           result = node.succeed(
 532               "openssl s_client -CAfile /tmp/ca.crt"
 533               f" -servername {domain} -connect {domain}:443 < /dev/null"
 534               " | openssl x509 -noout -ocsp_uri"
 535           )
 536           print("OCSP Responder URL:", result)
 537
 538           assert "${caDomain}:4002" in result.lower(), "OCSP Stapling check failed"
 539
 540
 541       @backoff.protect
 542       def download_ca_certs(node):
 543           node.succeed("curl https://${caDomain}:15000/roots/0 > /tmp/ca.crt")
 544           node.succeed("curl https://${caDomain}:15000/intermediate-keys/0 >> /tmp/ca.crt")
 545
 546
 547       @backoff.protect
 548       def set_a_record(node):
 549           node.succeed(
 550               'curl --data \'{"host": "${caDomain}", "addresses": ["${nodes.acme.networking.primaryIPAddress}"]}\' http://${dnsServerIP nodes}:8055/add-a'
 551           )
 552
 553
 554       start_all()
 555
 556       dnsserver.wait_for_unit("pebble-challtestsrv.service")
 557       client.wait_for_unit("default.target")
 558
 559       set_a_record(client)
 560
 561       acme.systemctl("start network-online.target")
 562       acme.wait_for_unit("network-online.target")
 563       acme.wait_for_unit("pebble.service")
 564
 565       download_ca_certs(client)
 566
 567       # Perform http-01 w/ lego test first
 568       with subtest("Can request certificate with Lego's built in web server"):
 569           switch_to(webserver, "http01lego")
 570           webserver.wait_for_unit("acme-finished-http.example.test.target")
 571           check_fullchain(webserver, "http.example.test")
 572           check_issuer(webserver, "http.example.test", "pebble")
 573
 574       # Perform account hash test
 575       with subtest("Assert that account hash didn't unexpectedly change"):
 576           hash = webserver.succeed("ls /var/lib/acme/.lego/accounts/")
 577           print("Account hash: " + hash)
 578           assert hash.strip() == "d590213ed52603e9128d"
 579
 580       # Perform renewal test
 581       with subtest("Can renew certificates when they expire"):
 582           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 583           switch_to(webserver, "renew")
 584           webserver.wait_for_unit("acme-finished-http.example.test.target")
 585           check_fullchain(webserver, "http.example.test")
 586           check_issuer(webserver, "http.example.test", "pebble")
 587           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 588           assert hash != hash_after
 589
 590       # Perform account change test
 591       with subtest("Handles email change correctly"):
 592           hash = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 593           switch_to(webserver, "accountchange")
 594           webserver.wait_for_unit("acme-finished-http.example.test.target")
 595           check_fullchain(webserver, "http.example.test")
 596           check_issuer(webserver, "http.example.test", "pebble")
 597           hash_after = webserver.succeed("sha256sum /var/lib/acme/http.example.test/cert.pem")
 598           # Has to do a full run to register account, which creates new certs.
 599           assert hash != hash_after
 600
 601       # Perform general tests
 602       switch_to(webserver, "general")
 603
 604       with subtest("Can request certificate with HTTP-01 challenge"):
 605           webserver.wait_for_unit("acme-finished-a.example.test.target")
 606           check_fullchain(webserver, "a.example.test")
 607           check_issuer(webserver, "a.example.test", "pebble")
 608           webserver.wait_for_unit("nginx.service")
 609           check_connection(client, "a.example.test")
 610
 611       with subtest("Runs 1 cert for account creation before others"):
 612           webserver.wait_for_unit("acme-finished-b.example.test.target")
 613           webserver.wait_for_unit("acme-finished-c.example.test.target")
 614           check_connection(client, "b.example.test")
 615           check_connection(client, "c.example.test")
 616
 617       with subtest("Certificates and accounts have safe + valid permissions"):
 618           # Nginx will set the group appropriately when enableACME is used
 619           group = "nginx"
 620           webserver.succeed(
 621               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 622           )
 623           webserver.succeed(
 624               f"test $(stat -L -c '%a %U %G' /var/lib/acme/.lego/a.example.test/**/a.example.test* | tee /dev/stderr | grep '600 acme {group}' | wc -l) -eq 4"
 625           )
 626           webserver.succeed(
 627               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test | tee /dev/stderr | grep '750 acme {group}' | wc -l) -eq 1"
 628           )
 629           webserver.succeed(
 630               f"test $(find /var/lib/acme/accounts -type f -exec stat -L -c '%a %U %G' {{}} \\; | tee /dev/stderr | grep -v '600 acme {group}' | wc -l) -eq 0"
 631           )
 632
 633       # Selfsigned certs tests happen late so we aren't fighting the system init triggering cert renewal
 634       with subtest("Can generate valid selfsigned certs"):
 635           webserver.succeed("systemctl clean acme-a.example.test.service --what=state")
 636           webserver.succeed("systemctl start acme-selfsigned-a.example.test.service")
 637           check_fullchain(webserver, "a.example.test")
 638           check_issuer(webserver, "a.example.test", "minica")
 639           # Check selfsigned permissions
 640           webserver.succeed(
 641               f"test $(stat -L -c '%a %U %G' /var/lib/acme/a.example.test/*.pem | tee /dev/stderr | grep '640 acme {group}' | wc -l) -eq 5"
 642           )
 643           # Will succeed if nginx can load the certs
 644           webserver.succeed("systemctl start nginx-config-reload.service")
 645
 646       with subtest("Correctly implements OCSP stapling"):
 647           switch_to(webserver, "ocsp_stapling")
 648           webserver.wait_for_unit("acme-finished-a.example.test.target")
 649           check_stapling(client, "a.example.test")
 650
 651       with subtest("Can request certificate with HTTP-01 using lego's internal web server"):
 652           switch_to(webserver, "lego_server")
 653           webserver.wait_for_unit("acme-finished-lego.example.test.target")
 654           webserver.wait_for_unit("nginx.service")
 655           webserver.succeed("echo HENLO && systemctl cat nginx.service")
 656           webserver.succeed('test "$(stat -c \'%U\' /var/lib/acme/* | uniq)" = "root"')
 657           check_connection(client, "a.example.test")
 658           check_connection(client, "lego.example.test")
 659
 660       with subtest("Can request certificate with HTTP-01 when nginx startup is delayed"):
 661           webserver.execute("systemctl stop nginx")
 662           switch_to(webserver, "slow_startup")
 663           webserver.wait_for_unit("acme-finished-slow.example.test.target")
 664           check_issuer(webserver, "slow.example.test", "pebble")
 665           webserver.wait_for_unit("nginx.service")
 666           check_connection(client, "slow.example.test")
 667
 668       with subtest("Can limit concurrency of running renewals"):
 669           switch_to(webserver, "concurrency_limit")
 670           webserver.wait_for_unit("acme-finished-f.example.test.target")
 671           webserver.wait_for_unit("acme-finished-g.example.test.target")
 672           webserver.wait_for_unit("acme-finished-h.example.test.target")
 673           check_connection(client, "f.example.test")
 674           check_connection(client, "g.example.test")
 675           check_connection(client, "h.example.test")
 676
 677       with subtest("Works with caddy"):
 678           switch_to(webserver, "caddy")
 679           webserver.wait_for_unit("acme-finished-example.test.target")
 680           webserver.wait_for_unit("caddy.service")
 681           # FIXME reloading caddy is not sufficient to load new certs.
 682           # Restart it manually until this is fixed.
 683           webserver.succeed("systemctl restart caddy.service")
 684           check_connection(client, "a.example.test")
 685
 686       with subtest("security.acme changes reflect on caddy"):
 687           switch_to(webserver, "caddy_change_acme_conf")
 688           webserver.wait_for_unit("acme-finished-example.test.target")
 689           webserver.wait_for_unit("caddy.service")
 690           # FIXME reloading caddy is not sufficient to load new certs.
 691           # Restart it manually until this is fixed.
 692           webserver.succeed("systemctl restart caddy.service")
 693           check_connection_key_bits(client, "a.example.test", "384")
 694
 695       common_domains = ["http", "dns", "wildcard"]
 696       for server, logsrc, domains in [
 697           ("nginx", "journalctl -n 30 -u nginx.service", common_domains + ["different-key"]),
 698           ("httpd", "tail -n 30 /var/log/httpd/*.log", common_domains),
 699       ]:
 700           wait_for_server = lambda: webserver.wait_for_unit(f"{server}.service")
 701           with subtest(f"Works with {server}"):
 702               try:
 703                   switch_to(webserver, server)
 704                   for domain in domains:
 705                       if domain != "wildcard":
 706                           webserver.wait_for_unit(
 707                               f"acme-finished-{server}-{domain}.example.test.target"
 708                           )
 709               except Exception as err:
 710                   _, output = webserver.execute(
 711                       f"{logsrc} && ls -al /var/lib/acme/acme-challenge"
 712                   )
 713                   print(output)
 714                   raise err
 715
 716               wait_for_server()
 717
 718               for domain in domains:
 719                   if domain != "wildcard":
 720                       check_issuer(webserver, f"{server}-{domain}.example.test", "pebble")
 721               for domain in domains:
 722                   check_connection(client, f"{server}-{domain}.example.test")
 723                   check_connection(client, f"{server}-{domain}-alias.example.test")
 724
 725           test_domain = f"{server}-{domains[0]}.example.test"
 726
 727           with subtest(f"Can reload {server} when timer triggers renewal"):
 728               # Switch to selfsigned first
 729               webserver.succeed(f"systemctl clean acme-{test_domain}.service --what=state")
 730               webserver.succeed(f"systemctl start acme-selfsigned-{test_domain}.service")
 731               check_issuer(webserver, test_domain, "minica")
 732               webserver.succeed(f"systemctl start {server}-config-reload.service")
 733               webserver.succeed(f"systemctl start test-renew-{server}.target")
 734               check_issuer(webserver, test_domain, "pebble")
 735               check_connection(client, test_domain)
 736
 737           with subtest("Can remove an alias from a domain + cert is updated"):
 738               test_alias = f"{server}-{domains[0]}-alias.example.test"
 739               switch_to(webserver, f"{server}_remove_alias")
 740               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 741               wait_for_server()
 742               check_connection(client, test_domain)
 743               rc, _s = client.execute(
 744                   f"openssl s_client -CAfile /tmp/ca.crt -connect {test_alias}:443"
 745                   " </dev/null 2>/dev/null | openssl x509 -noout -text"
 746                   f" | grep DNS: | grep {test_alias}"
 747               )
 748               assert rc > 0, "Removed extraDomainName was not removed from the cert"
 749
 750           with subtest("security.acme changes reflect on web server"):
 751               # Switch back to normal server config first, reset everything.
 752               switch_to(webserver, server)
 753               wait_for_server()
 754               switch_to(webserver, f"{server}_change_acme_conf")
 755               webserver.wait_for_unit(f"acme-finished-{test_domain}.target")
 756               wait_for_server()
 757               check_connection_key_bits(client, test_domain, "384")
 758
 759       # Perform http-01 w/ lego test again, but using the pre-24.05 account hashing
 760       # (see https://github.com/NixOS/nixpkgs/pull/317257)
 761       with subtest("Check account hashing compatibility with pre-24.05 settings"):
 762           webserver.succeed("rm -rf /var/lib/acme/.lego/accounts/*")
 763           switch_to(webserver, "http01lego_legacyAccountHash", allow_fail=True)
 764           # unit is failed, but in a way that this throws no exception:
 765           try:
 766             webserver.wait_for_unit("acme-finished-http.example.test.target")
 767           except Exception:
 768             # The unit is allowed – or even expected – to fail due to not being able to
 769             # reach the actual letsencrypt server. We only use it for serialising the
 770             # test execution, such that the account check is done after the service run
 771             # involving the account creation has been executed at least once.
 772             pass
 773           hash = webserver.succeed("ls /var/lib/acme/.lego/accounts/")
 774           print("Account hash: " + hash)
 775           assert hash.strip() == "1ccf607d9aa280e9af00"
 776     '';
 777 }