nixos/tests/prometheus-exporters.nix

   1 { system ? builtins.currentSystem
   2 , config ? { }
   3 , pkgs ? import ../.. { inherit system config; }
   4 }:
   5
   6 let
   7   inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
   8   inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
   9     removeSuffix replaceStrings singleton splitString makeBinPath;
  10
  11   /*
  12     * The attrset `exporterTests` contains one attribute
  13     * for each exporter test. Each of these attributes
  14     * is expected to be an attrset containing:
  15     *
  16     *  `exporterConfig`:
  17     *    this attribute set contains config for the exporter itself
  18     *
  19     *  `exporterTest`
  20     *    this attribute set contains test instructions
  21     *
  22     *  `metricProvider` (optional)
  23     *    this attribute contains additional machine config
  24     *
  25     *  `nodeName` (optional)
  26     *    override an incompatible testnode name
  27     *
  28     *  Example:
  29     *    exporterTests.<exporterName> = {
  30     *      exporterConfig = {
  31     *        enable = true;
  32     *      };
  33     *      metricProvider = {
  34     *        services.<metricProvider>.enable = true;
  35     *      };
  36     *      exporterTest = ''
  37     *        wait_for_unit("prometheus-<exporterName>-exporter.service")
  38     *        wait_for_open_port(1234)
  39     *        succeed("curl -sSf 'localhost:1234/metrics'")
  40     *      '';
  41     *    };
  42     *
  43     *  # this would generate the following test config:
  44     *
  45     *    nodes.<exporterName> = {
  46     *      services.prometheus.<exporterName> = {
  47     *        enable = true;
  48     *      };
  49     *      services.<metricProvider>.enable = true;
  50     *    };
  51     *
  52     *    testScript = ''
  53     *      <exporterName>.start()
  54     *      <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
  55     *      <exporterName>.wait_for_open_port(1234)
  56     *      <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
  57     *      <exporterName>.shutdown()
  58     *    '';
  59   */
  60
  61   exporterTests = {
  62     apcupsd = {
  63       exporterConfig = {
  64         enable = true;
  65       };
  66       metricProvider = {
  67         services.apcupsd.enable = true;
  68       };
  69       exporterTest = ''
  70         wait_for_unit("apcupsd.service")
  71         wait_for_open_port(3551)
  72         wait_for_unit("prometheus-apcupsd-exporter.service")
  73         wait_for_open_port(9162)
  74         succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
  75       '';
  76     };
  77
  78     artifactory = {
  79       exporterConfig = {
  80         enable = true;
  81         artiUsername = "artifactory-username";
  82         artiPassword = "artifactory-password";
  83       };
  84       exporterTest = ''
  85         wait_for_unit("prometheus-artifactory-exporter.service")
  86         wait_for_open_port(9531)
  87         succeed(
  88             "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
  89         )
  90       '';
  91     };
  92
  93     bind = {
  94       exporterConfig = {
  95         enable = true;
  96       };
  97       metricProvider = {
  98         services.bind.enable = true;
  99         services.bind.extraConfig = ''
 100           statistics-channels {
 101             inet 127.0.0.1 port 8053 allow { localhost; };
 102           };
 103         '';
 104       };
 105       exporterTest = ''
 106         wait_for_unit("prometheus-bind-exporter.service")
 107         wait_for_open_port(9119)
 108         succeed(
 109             "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
 110         )
 111       '';
 112     };
 113
 114     bird = {
 115       exporterConfig = {
 116         enable = true;
 117       };
 118       metricProvider = {
 119         services.bird2.enable = true;
 120         services.bird2.config = ''
 121           router id 127.0.0.1;
 122
 123           protocol kernel MyObviousTestString {
 124             ipv4 {
 125               import all;
 126               export none;
 127             };
 128           }
 129
 130           protocol device {
 131           }
 132         '';
 133       };
 134       exporterTest = ''
 135         wait_for_unit("prometheus-bird-exporter.service")
 136         wait_for_open_port(9324)
 137         wait_until_succeeds(
 138             "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
 139         )
 140       '';
 141     };
 142
 143     bitcoin = {
 144       exporterConfig = {
 145         enable = true;
 146         rpcUser = "bitcoinrpc";
 147         rpcPasswordFile = pkgs.writeText "password" "hunter2";
 148       };
 149       metricProvider = {
 150         services.bitcoind.default.enable = true;
 151         services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
 152       };
 153       exporterTest = ''
 154         wait_for_unit("prometheus-bitcoin-exporter.service")
 155         wait_for_unit("bitcoind-default.service")
 156         wait_for_open_port(9332)
 157         succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
 158       '';
 159     };
 160
 161     blackbox = {
 162       exporterConfig = {
 163         enable = true;
 164         configFile = pkgs.writeText "config.yml" (builtins.toJSON {
 165           modules.icmp_v6 = {
 166             prober = "icmp";
 167             icmp.preferred_ip_protocol = "ip6";
 168           };
 169         });
 170       };
 171       exporterTest = ''
 172         wait_for_unit("prometheus-blackbox-exporter.service")
 173         wait_for_open_port(9115)
 174         succeed(
 175             "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
 176         )
 177       '';
 178     };
 179
 180     borgmatic = {
 181       exporterConfig = {
 182         enable = true;
 183         user = "root";
 184       };
 185       metricProvider = {
 186         services.borgmatic.enable = true;
 187         services.borgmatic.settings.source_directories = [ "/home" ];
 188         services.borgmatic.settings.repositories = [ { label = "local"; path = "/var/backup"; } ];
 189         services.borgmatic.settings.keep_daily = 10;
 190       };
 191       exporterTest = ''
 192         succeed("borgmatic rcreate -e none")
 193         succeed("borgmatic")
 194         wait_for_unit("prometheus-borgmatic-exporter.service")
 195         wait_for_open_port(9996)
 196         succeed("curl -sSf localhost:9996/metrics | grep 'borg_total_backups{repository=\"/var/backup\"} 1'")
 197       '';
 198     };
 199
 200     collectd = {
 201       exporterConfig = {
 202         enable = true;
 203         extraFlags = [ "--web.collectd-push-path /collectd" ];
 204       };
 205       exporterTest = let postData = replaceStrings [ "\n" ] [ "" ] ''
 206         [{
 207           "values":[23],
 208           "dstypes":["gauge"],
 209           "type":"gauge",
 210           "interval":1000,
 211           "host":"testhost",
 212           "plugin":"testplugin",
 213           "time":DATE
 214         }]
 215       ''; in
 216         ''
 217           wait_for_unit("prometheus-collectd-exporter.service")
 218           wait_for_open_port(9103)
 219           succeed(
 220               'echo \'${postData}\'> /tmp/data.json'
 221           )
 222           succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
 223           succeed(
 224               "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
 225           )
 226           succeed(
 227               "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
 228           )
 229         '';
 230     };
 231
 232     deluge = {
 233       exporterConfig = {
 234         enable = true;
 235         port = 1234;
 236         listenAddress = "127.0.0.1";
 237
 238         delugeUser = "user";
 239         delugePort = 2345;
 240         delugePasswordFile = pkgs.writeText "password" "weak_password";
 241       };
 242       metricProvider = {
 243         services.deluge.enable = true;
 244         services.deluge.declarative = true;
 245         services.deluge.config.daemon_port = 2345;
 246         services.deluge.authFile = pkgs.writeText "authFile" ''
 247         localclient:abcdef:10
 248         user:weak_password:10
 249         '';
 250       };
 251       exporterTest = ''
 252         wait_for_unit("deluged.service")
 253         wait_for_open_port(2345)
 254         wait_for_unit("prometheus-deluge-exporter.service")
 255         wait_for_open_port(1234)
 256         succeed("curl -sSf http://localhost:1234 | grep 'deluge_torrents'")
 257       '';
 258     };
 259
 260     dnsmasq = {
 261       exporterConfig = {
 262         enable = true;
 263         leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
 264       };
 265       metricProvider = {
 266         services.dnsmasq.enable = true;
 267       };
 268       exporterTest = ''
 269         wait_for_unit("dnsmasq.service")
 270         wait_for_open_port(53)
 271         wait_for_file("/var/lib/dnsmasq/dnsmasq.leases")
 272         wait_for_unit("prometheus-dnsmasq-exporter.service")
 273         wait_for_open_port(9153)
 274         succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
 275       '';
 276     };
 277
 278     dnssec = {
 279       exporterConfig = {
 280         enable = true;
 281         configuration = {
 282           records = [
 283             {
 284               zone = "example.com";
 285               record = "@";
 286               type = "SOA";
 287             }
 288           ];
 289         };
 290         resolvers = [ "127.0.0.1:53" ];
 291       };
 292       metricProvider = {
 293         services.knot = {
 294           enable = true;
 295           settingsFile = pkgs.writeText "knot.conf" ''
 296             server:
 297               listen: 127.0.0.1@53
 298             template:
 299               - id: default
 300                 storage: ${pkgs.buildEnv {
 301                   name = "zones";
 302                   paths = [(pkgs.writeTextDir "example.com.zone" ''
 303                     @ SOA ns1.example.com. noc.example.com. 2024032401 86400 7200 3600000 172800
 304                     @       NS      ns1
 305                     ns1     A       192.168.0.1
 306                   '')];
 307                 }}
 308                 zonefile-load: difference
 309                 zonefile-sync: -1
 310             zone:
 311               - domain: example.com
 312                 file: example.com.zone
 313                 dnssec-signing: on
 314           '';
 315         };
 316       };
 317       exporterTest = ''
 318         wait_for_unit("knot.service")
 319         wait_for_open_port(53)
 320         wait_for_unit("prometheus-dnssec-exporter.service")
 321         wait_for_open_port(9204)
 322         succeed("curl -sSf http://localhost:9204/metrics | grep 'example.com'")
 323       '';
 324     };
 325
 326     # Access to WHOIS server is required to properly test this exporter, so
 327     # just perform basic sanity check that the exporter is running and returns
 328     # a failure.
 329     domain = {
 330       exporterConfig = {
 331         enable = true;
 332       };
 333       exporterTest = ''
 334         wait_for_unit("prometheus-domain-exporter.service")
 335         wait_for_open_port(9222)
 336         succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
 337       '';
 338     };
 339
 340     dovecot = {
 341       exporterConfig = {
 342         enable = true;
 343         scopes = [ "global" ];
 344         socketPath = "/var/run/dovecot2/old-stats";
 345         user = "root"; # <- don't use user root in production
 346       };
 347       metricProvider = {
 348         services.dovecot2.enable = true;
 349       };
 350       exporterTest = ''
 351         wait_for_unit("prometheus-dovecot-exporter.service")
 352         wait_for_open_port(9166)
 353         succeed(
 354             "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
 355         )
 356       '';
 357     };
 358
 359     exportarr-sonarr = let apikey = "eccff6a992bc2e4b88e46d064b26bb4e"; in {
 360       nodeName = "exportarr_sonarr";
 361       exporterConfig = {
 362         enable = true;
 363         url = "http://127.0.0.1:8989";
 364         apiKeyFile = pkgs.writeText "dummy-api-key" apikey;
 365       };
 366       metricProvider = {
 367         services.sonarr.enable = true;
 368         systemd.services.sonarr.serviceConfig.ExecStartPre =
 369           let
 370             sonarr_config = pkgs.writeText "config.xml" ''
 371               <Config>
 372                 <LogLevel>info</LogLevel>
 373                 <EnableSsl>False</EnableSsl>
 374                 <Port>8989</Port>
 375                 <SslPort>9898</SslPort>
 376                 <UrlBase></UrlBase>
 377                 <BindAddress>*</BindAddress>
 378                 <ApiKey>${apikey}</ApiKey>
 379                 <AuthenticationMethod>None</AuthenticationMethod>
 380                 <UpdateMechanism>BuiltIn</UpdateMechanism>
 381                 <Branch>main</Branch>
 382                 <InstanceName>Sonarr</InstanceName>
 383               </Config>
 384             '';
 385           in
 386           [
 387             ''${pkgs.coreutils}/bin/install -D -m 777 ${sonarr_config} -T /var/lib/sonarr/.config/NzbDrone/config.xml''
 388           ];
 389       };
 390       exporterTest = ''
 391         wait_for_unit("sonarr.service")
 392         wait_for_open_port(8989)
 393         wait_for_unit("prometheus-exportarr-sonarr-exporter.service")
 394         wait_for_open_port(9708)
 395         succeed("curl -sSf http://localhost:9708/metrics | grep sonarr_series_total")
 396       '';
 397     };
 398
 399     fastly = {
 400       exporterConfig = {
 401         enable = true;
 402         tokenPath = pkgs.writeText "token" "abc123";
 403       };
 404
 405       exporterTest = ''
 406         wait_for_unit("prometheus-fastly-exporter.service")
 407         wait_for_open_port(9118)
 408       '';
 409     };
 410
 411     fritzbox = {
 412       # TODO add proper test case
 413       exporterConfig = {
 414         enable = true;
 415       };
 416       exporterTest = ''
 417         wait_for_unit("prometheus-fritzbox-exporter.service")
 418         wait_for_open_port(9133)
 419         succeed(
 420             "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
 421         )
 422       '';
 423     };
 424
 425     graphite = {
 426       exporterConfig = {
 427         enable = true;
 428         port = 9108;
 429         graphitePort = 9109;
 430         mappingSettings.mappings = [{
 431           match = "test.*.*";
 432           name = "testing";
 433           labels = {
 434             protocol = "$1";
 435             author = "$2";
 436           };
 437         }];
 438       };
 439       exporterTest = ''
 440         wait_for_unit("prometheus-graphite-exporter.service")
 441         wait_for_open_port(9108)
 442         wait_for_open_port(9109)
 443         succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
 444         succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
 445       '';
 446     };
 447
 448     idrac = {
 449       exporterConfig = {
 450         enable = true;
 451         port = 9348;
 452         configuration = {
 453           hosts = {
 454             default = { username = "username"; password = "password"; };
 455           };
 456         };
 457       };
 458       exporterTest = ''
 459         wait_for_unit("prometheus-idrac-exporter.service")
 460         wait_for_open_port(9348)
 461         wait_until_succeeds("curl localhost:9348")
 462       '';
 463     };
 464
 465     influxdb = {
 466       exporterConfig = {
 467         enable = true;
 468         sampleExpiry = "3s";
 469       };
 470       exporterTest = ''
 471         wait_for_unit("prometheus-influxdb-exporter.service")
 472         wait_for_open_port(9122)
 473         succeed(
 474           "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
 475         )
 476         succeed(
 477           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 478         )
 479         execute("sleep 5")
 480         fail(
 481           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 482         )
 483       '';
 484     };
 485
 486     ipmi = {
 487       exporterConfig = {
 488         enable = true;
 489       };
 490       exporterTest = ''
 491         wait_for_unit("prometheus-ipmi-exporter.service")
 492         wait_for_open_port(9290)
 493         succeed(
 494           "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
 495         )
 496       '';
 497     };
 498
 499     jitsi = {
 500       exporterConfig = {
 501         enable = true;
 502       };
 503       metricProvider = {
 504         systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
 505         services.jitsi-videobridge = {
 506           enable = true;
 507           colibriRestApi = true;
 508         };
 509       };
 510       exporterTest = ''
 511         wait_for_unit("jitsi-videobridge2.service")
 512         wait_for_open_port(8080)
 513         wait_for_unit("prometheus-jitsi-exporter.service")
 514         wait_for_open_port(9700)
 515         wait_until_succeeds(
 516             'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
 517         )
 518         succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
 519       '';
 520     };
 521
 522     json = {
 523       exporterConfig = {
 524         enable = true;
 525         configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
 526           modules = {
 527             default = {
 528               metrics = [
 529                 { name = "json_test_metric"; path = "{ .test }"; }
 530               ];
 531             };
 532           };
 533         });
 534       };
 535       metricProvider = {
 536         systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
 537         services.nginx = {
 538           enable = true;
 539           virtualHosts.localhost.locations."/".extraConfig = ''
 540             return 200 "{\"test\":1}";
 541           '';
 542         };
 543       };
 544       exporterTest = ''
 545         wait_for_unit("nginx.service")
 546         wait_for_open_port(80)
 547         wait_for_unit("prometheus-json-exporter.service")
 548         wait_for_open_port(7979)
 549         succeed(
 550             "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
 551         )
 552       '';
 553     };
 554
 555     knot = {
 556       exporterConfig = {
 557         enable = true;
 558       };
 559       metricProvider = {
 560         services.knot = {
 561           enable = true;
 562           extraArgs = [ "-v" ];
 563           settingsFile = pkgs.writeText "knot.conf" ''
 564             server:
 565               listen: 127.0.0.1@53
 566
 567             template:
 568               - id: default
 569                 global-module: mod-stats
 570                 dnssec-signing: off
 571                 zonefile-sync: -1
 572                 zonefile-load: difference
 573                 storage: ${pkgs.buildEnv {
 574                   name = "foo";
 575                   paths = [
 576                     (pkgs.writeTextDir "test.zone" ''
 577                       @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
 578                       @       NS      ns1
 579                       @       NS      ns2
 580                       ns1     A       192.168.0.1
 581                     '')
 582                   ];
 583                 }}
 584
 585             mod-stats:
 586               - id: custom
 587                 edns-presence: on
 588                 query-type: on
 589
 590             zone:
 591               - domain: test
 592                 file: test.zone
 593                 module: mod-stats/custom
 594           '';
 595         };
 596       };
 597       exporterTest = ''
 598         wait_for_unit("knot.service")
 599         wait_for_unit("prometheus-knot-exporter.service")
 600         wait_for_open_port(9433)
 601         succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
 602       '';
 603     };
 604
 605     keylight = {
 606       # A hardware device is required to properly test this exporter, so just
 607       # perform a couple of basic sanity checks that the exporter is running
 608       # and requires a target, but cannot reach a specified target.
 609       exporterConfig = {
 610         enable = true;
 611       };
 612       exporterTest = ''
 613         wait_for_unit("prometheus-keylight-exporter.service")
 614         wait_for_open_port(9288)
 615         succeed(
 616             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
 617         )
 618         succeed(
 619             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
 620         )
 621       '';
 622     };
 623
 624     lnd = {
 625       exporterConfig = {
 626         enable = true;
 627         lndTlsPath = "/var/lib/lnd/tls.cert";
 628         lndMacaroonDir = "/var/lib/lnd";
 629         extraFlags = [ "--lnd.network=regtest" ];
 630       };
 631       metricProvider = {
 632         systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
 633         systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
 634         services.bitcoind.regtest = {
 635           enable = true;
 636           extraConfig = ''
 637             rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
 638             zmqpubrawblock=tcp://127.0.0.1:28332
 639             zmqpubrawtx=tcp://127.0.0.1:28333
 640             # https://github.com/lightningnetwork/lnd/issues/9163
 641             deprecatedrpc=warnings
 642           '';
 643           extraCmdlineOptions = [ "-regtest" ];
 644         };
 645         systemd.services.lnd = {
 646           serviceConfig.ExecStart = ''
 647             ${pkgs.lnd}/bin/lnd \
 648               --datadir=/var/lib/lnd \
 649               --tlscertpath=/var/lib/lnd/tls.cert \
 650               --tlskeypath=/var/lib/lnd/tls.key \
 651               --logdir=/var/log/lnd \
 652               --bitcoin.active \
 653               --bitcoin.regtest \
 654               --bitcoin.node=bitcoind \
 655               --bitcoind.rpcuser=bitcoinrpc \
 656               --bitcoind.rpcpass=hunter2 \
 657               --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
 658               --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
 659               --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
 660           '';
 661           serviceConfig.StateDirectory = "lnd";
 662           wantedBy = [ "multi-user.target" ];
 663           after = [ "network.target" ];
 664         };
 665         # initialize wallet, creates macaroon needed by exporter
 666         systemd.services.lnd.postStart = ''
 667           ${pkgs.curl}/bin/curl \
 668             --retry 20 \
 669             --retry-delay 1 \
 670             --retry-connrefused \
 671             --cacert /var/lib/lnd/tls.cert \
 672             -X GET \
 673             https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
 674           ${pkgs.curl}/bin/curl \
 675             --retry 20 \
 676             --retry-delay 1 \
 677             --retry-connrefused \
 678             --cacert /var/lib/lnd/tls.cert \
 679             -X POST \
 680             -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
 681             https://localhost:8080/v1/initwallet
 682         '';
 683       };
 684       exporterTest = ''
 685         wait_for_unit("lnd.service")
 686         wait_for_open_port(10009)
 687         wait_for_unit("prometheus-lnd-exporter.service")
 688         wait_for_open_port(9092)
 689         succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
 690       '';
 691     };
 692
 693     mail = {
 694       exporterConfig = {
 695         enable = true;
 696         configuration = {
 697           monitoringInterval = "2s";
 698           mailCheckTimeout = "10s";
 699           servers = [{
 700             name = "testserver";
 701             server = "localhost";
 702             port = 25;
 703             from = "mail-exporter@localhost";
 704             to = "mail-exporter@localhost";
 705             detectionDir = "/var/spool/mail/mail-exporter/new";
 706           }];
 707         };
 708       };
 709       metricProvider = {
 710         services.postfix.enable = true;
 711         systemd.services.prometheus-mail-exporter = {
 712           after = [ "postfix.service" ];
 713           requires = [ "postfix.service" ];
 714           serviceConfig = {
 715             ExecStartPre = [
 716               "${pkgs.writeShellScript "create-maildir" ''
 717                 mkdir -p -m 0700 mail-exporter/new
 718               ''}"
 719             ];
 720             ProtectHome = true;
 721             ReadOnlyPaths = "/";
 722             ReadWritePaths = "/var/spool/mail";
 723             WorkingDirectory = "/var/spool/mail";
 724           };
 725         };
 726         users.users.mailexporter = {
 727           isSystemUser = true;
 728           group = "mailexporter";
 729         };
 730         users.groups.mailexporter = {};
 731       };
 732       exporterTest = ''
 733         wait_for_unit("postfix.service")
 734         wait_for_unit("prometheus-mail-exporter.service")
 735         wait_for_open_port(9225)
 736         wait_until_succeeds(
 737             "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
 738         )
 739       '';
 740     };
 741
 742     mikrotik = {
 743       exporterConfig = {
 744         enable = true;
 745         extraFlags = [ "-timeout=1s" ];
 746         configuration = {
 747           devices = [
 748             {
 749               name = "router";
 750               address = "192.168.42.48";
 751               user = "prometheus";
 752               password = "shh";
 753             }
 754           ];
 755           features = {
 756             bgp = true;
 757             dhcp = true;
 758             dhcpl = true;
 759             dhcpv6 = true;
 760             health = true;
 761             routes = true;
 762             poe = true;
 763             pools = true;
 764             optics = true;
 765             w60g = true;
 766             wlansta = true;
 767             wlanif = true;
 768             monitor = true;
 769             ipsec = true;
 770           };
 771         };
 772       };
 773       exporterTest = ''
 774         wait_for_unit("prometheus-mikrotik-exporter.service")
 775         wait_for_open_port(9436)
 776         succeed(
 777             "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
 778         )
 779       '';
 780     };
 781
 782     modemmanager = {
 783       exporterConfig = {
 784         enable = true;
 785         refreshRate = "10s";
 786       };
 787       metricProvider = {
 788         # ModemManager is installed when NetworkManager is enabled. Ensure it is
 789         # started and is wanted by NM and the exporter to start everything up
 790         # in the right order.
 791         networking.networkmanager.enable = true;
 792         systemd.services.ModemManager = {
 793           enable = true;
 794           wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
 795         };
 796       };
 797       exporterTest = ''
 798         wait_for_unit("ModemManager.service")
 799         wait_for_unit("prometheus-modemmanager-exporter.service")
 800         wait_for_open_port(9539)
 801         succeed(
 802             "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
 803         )
 804       '';
 805     };
 806
 807     mqtt = {
 808       exporterConfig = {
 809         enable = true;
 810         environmentFile = pkgs.writeText "mqtt-exporter-envfile" ''
 811           MQTT_PASSWORD=testpassword
 812         '';
 813       };
 814       metricProvider = {
 815         services.mosquitto = {
 816           enable = true;
 817           listeners = [{
 818             users.exporter = {
 819               acl = [ "read #" ];
 820               passwordFile = pkgs.writeText "mosquitto-password" "testpassword";
 821             };
 822           }];
 823         };
 824         systemd.services.prometheus-mqtt-exporter ={
 825           wants = [ "mosquitto.service" ];
 826           after = [ "mosquitto.service" ];
 827         };
 828       };
 829       exporterTest = ''
 830         wait_for_unit("mosquitto.service")
 831         wait_for_unit("prometheus-mqtt-exporter.service")
 832         wait_for_open_port(9000)
 833         succeed(
 834           "curl -sSf http://localhost:9000/metrics | grep '^python_info'"
 835         )
 836       '';
 837     };
 838
 839     mysqld = {
 840       exporterConfig = {
 841         enable = true;
 842         runAsLocalSuperUser = true;
 843         configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
 844           [client]
 845           user = exporter
 846           password = snakeoilpassword
 847         '';
 848       };
 849       metricProvider = {
 850         services.mysql = {
 851           enable = true;
 852           package = pkgs.mariadb;
 853           initialScript = pkgs.writeText "mysql-init-script.sql" ''
 854             CREATE USER 'exporter'@'localhost'
 855             IDENTIFIED BY 'snakeoilpassword'
 856             WITH MAX_USER_CONNECTIONS 3;
 857             GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
 858           '';
 859         };
 860       };
 861       exporterTest = ''
 862         wait_for_unit("prometheus-mysqld-exporter.service")
 863         wait_for_open_port(9104)
 864         wait_for_unit("mysql.service")
 865         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 866         systemctl("stop mysql.service")
 867         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
 868         systemctl("start mysql.service")
 869         wait_for_unit("mysql.service")
 870         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 871       '';
 872     };
 873
 874     nextcloud = {
 875       exporterConfig = {
 876         enable = true;
 877         passwordFile = "/var/nextcloud-pwfile";
 878         url = "http://localhost";
 879       };
 880       metricProvider = {
 881         systemd.services.nc-pwfile =
 882           let
 883             passfile = (pkgs.writeText "pwfile" "snakeoilpw");
 884           in
 885           {
 886             requiredBy = [ "prometheus-nextcloud-exporter.service" ];
 887             before = [ "prometheus-nextcloud-exporter.service" ];
 888             serviceConfig.ExecStart = ''
 889               ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
 890             '';
 891           };
 892         services.nginx = {
 893           enable = true;
 894           virtualHosts."localhost" = {
 895             basicAuth.nextcloud-exporter = "snakeoilpw";
 896             locations."/" = {
 897               root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
 898               tryFiles = "/negative-space.json =404";
 899             };
 900           };
 901         };
 902       };
 903       exporterTest = ''
 904         wait_for_unit("nginx.service")
 905         wait_for_unit("prometheus-nextcloud-exporter.service")
 906         wait_for_open_port(9205)
 907         succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
 908       '';
 909     };
 910
 911     nginx = {
 912       exporterConfig = {
 913         enable = true;
 914         constLabels = [ "foo=bar" ];
 915       };
 916       metricProvider = {
 917         services.nginx = {
 918           enable = true;
 919           statusPage = true;
 920           virtualHosts."test".extraConfig = "return 204;";
 921         };
 922       };
 923       exporterTest = ''
 924         wait_for_unit("nginx.service")
 925         wait_for_unit("prometheus-nginx-exporter.service")
 926         wait_for_open_port(9113)
 927         succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up{foo=\"bar\"} 1'")
 928       '';
 929     };
 930
 931     nginxlog = {
 932       exporterConfig = {
 933         enable = true;
 934         group = "nginx";
 935         settings = {
 936           namespaces = [
 937             {
 938               name = "filelogger";
 939               source = {
 940                 files = [ "/var/log/nginx/filelogger.access.log" ];
 941               };
 942             }
 943             {
 944               name = "syslogger";
 945               source = {
 946                 syslog = {
 947                   listen_address = "udp://127.0.0.1:10000";
 948                   format = "rfc3164";
 949                   tags = [ "nginx" ];
 950                 };
 951               };
 952             }
 953           ];
 954         };
 955       };
 956       metricProvider = {
 957         services.nginx = {
 958           enable = true;
 959           httpConfig = ''
 960             server {
 961               listen 80;
 962               server_name filelogger.local;
 963               access_log /var/log/nginx/filelogger.access.log;
 964             }
 965             server {
 966               listen 81;
 967               server_name syslogger.local;
 968               access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
 969             }
 970           '';
 971         };
 972       };
 973       exporterTest = ''
 974         wait_for_unit("nginx.service")
 975         wait_for_unit("prometheus-nginxlog-exporter.service")
 976         wait_for_open_port(9117)
 977         wait_for_open_port(80)
 978         wait_for_open_port(81)
 979         succeed("curl http://localhost")
 980         execute("sleep 1")
 981         succeed(
 982             "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
 983         )
 984         succeed("curl http://localhost:81")
 985         execute("sleep 1")
 986         succeed(
 987             "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
 988         )
 989       '';
 990     };
 991
 992     node = {
 993       exporterConfig = {
 994         enable = true;
 995       };
 996       exporterTest = ''
 997         wait_for_unit("prometheus-node-exporter.service")
 998         wait_for_open_port(9100)
 999         succeed(
1000             "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
1001         )
1002       '';
1003     };
1004
1005     pgbouncer = {
1006       exporterConfig = {
1007         enable = true;
1008         connectionEnvFile = "${pkgs.writeText "connstr-env" ''
1009           PGBOUNCER_EXPORTER_CONNECTION_STRING=postgres://admin@localhost:6432/pgbouncer?sslmode=disable
1010         ''}";
1011       };
1012
1013       metricProvider = {
1014         services.postgresql.enable = true;
1015         services.pgbouncer = {
1016           enable = true;
1017           settings = {
1018             pgbouncer = {
1019               listen_addr = "*";
1020               auth_type = "any";
1021               max_client_conn = 99;
1022               # https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
1023               ignore_startup_parameters = "extra_float_digits";
1024             };
1025             databases = {
1026               postgres = concatStringsSep " " [
1027                 "host=/run/postgresql"
1028                 "port=5432"
1029                 "auth_user=postgres"
1030                 "dbname=postgres"
1031               ];
1032             };
1033           };
1034         };
1035       };
1036       exporterTest = ''
1037         wait_for_unit("postgresql.service")
1038         wait_for_unit("pgbouncer.service")
1039         wait_for_unit("prometheus-pgbouncer-exporter.service")
1040         wait_for_open_port(9127)
1041         succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
1042         succeed(
1043             "curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
1044         )
1045       '';
1046     };
1047
1048     php-fpm = {
1049       nodeName = "php_fpm";
1050       exporterConfig = {
1051         enable = true;
1052         environmentFile = pkgs.writeTextFile {
1053           name = "/tmp/prometheus-php-fpm-exporter.env";
1054           text = ''
1055             PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
1056           '';
1057         };
1058       };
1059       metricProvider = {
1060         users.users."php-fpm-exporter" = {
1061           isSystemUser = true;
1062           group  = "php-fpm-exporter";
1063         };
1064         users.groups."php-fpm-exporter" = {};
1065         services.phpfpm.pools."php-fpm-exporter" = {
1066           user = "php-fpm-exporter";
1067           group = "php-fpm-exporter";
1068           settings = {
1069             "pm" = "dynamic";
1070             "pm.max_children" = 32;
1071             "pm.max_requests" = 500;
1072             "pm.start_servers" = 2;
1073             "pm.min_spare_servers" = 2;
1074             "pm.max_spare_servers" = 5;
1075             "pm.status_path" = "/status";
1076             "listen" = "127.0.0.1:9000";
1077             "listen.allowed_clients" = "127.0.0.1";
1078           };
1079           phpEnv."PATH" = makeBinPath [ pkgs.php ];
1080         };
1081       };
1082       exporterTest = ''
1083         wait_for_unit("phpfpm-php-fpm-exporter.service")
1084         wait_for_unit("prometheus-php-fpm-exporter.service")
1085         succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
1086       '';
1087     };
1088
1089     ping = {
1090       exporterConfig = {
1091         enable = true;
1092
1093         settings = {
1094           targets = [ {
1095             "localhost" = {
1096               alias = "local machine";
1097               env = "prod";
1098               type = "domain";
1099             };
1100           } {
1101             "127.0.0.1" = {
1102               alias = "local machine";
1103               type = "v4";
1104             };
1105           } {
1106             "::1" = {
1107               alias = "local machine";
1108               type = "v6";
1109             };
1110           } {
1111             "google.com" = {};
1112           } ];
1113           dns = {};
1114           ping = {
1115             interval = "2s";
1116             timeout = "3s";
1117             history-size = 42;
1118             payload-size = 56;
1119           };
1120           log = {
1121             level = "warn";
1122           };
1123         };
1124       };
1125
1126       exporterTest = ''
1127         wait_for_unit("prometheus-ping-exporter.service")
1128         wait_for_open_port(9427)
1129         succeed("curl -sSf http://localhost:9427/metrics | grep 'ping_up{.*} 1'")
1130       '';
1131     };
1132
1133     postfix = {
1134       exporterConfig = {
1135         enable = true;
1136       };
1137       metricProvider = {
1138         services.postfix.enable = true;
1139       };
1140       exporterTest = ''
1141         wait_for_unit("prometheus-postfix-exporter.service")
1142         wait_for_file("/var/lib/postfix/queue/public/showq")
1143         wait_for_open_port(9154)
1144         wait_until_succeeds(
1145             "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
1146         )
1147         succeed(
1148             "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
1149         )
1150         succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
1151       '';
1152     };
1153
1154     postgres = {
1155       exporterConfig = {
1156         enable = true;
1157         runAsLocalSuperUser = true;
1158       };
1159       metricProvider = {
1160         services.postgresql.enable = true;
1161       };
1162       exporterTest = ''
1163         wait_for_unit("prometheus-postgres-exporter.service")
1164         wait_for_open_port(9187)
1165         wait_for_unit("postgresql.service")
1166         succeed(
1167             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1168         )
1169         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1170         systemctl("stop postgresql.service")
1171         succeed(
1172             "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
1173         )
1174         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
1175         systemctl("start postgresql.service")
1176         wait_for_unit("postgresql.service")
1177         succeed(
1178             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1179         )
1180         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1181       '';
1182     };
1183
1184     process = {
1185       exporterConfig = {
1186         enable = true;
1187         settings.process_names = [
1188           # Remove nix store path from process name
1189           { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
1190         ];
1191       };
1192       exporterTest = ''
1193         wait_for_unit("prometheus-process-exporter.service")
1194         wait_for_open_port(9256)
1195         wait_until_succeeds(
1196             "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
1197                 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
1198             )
1199         )
1200       '';
1201     };
1202
1203     pve = let
1204       pveExporterEnvFile = pkgs.writeTextFile {
1205         name = "pve.env";
1206         text = ''
1207           PVE_USER="test_user@pam"
1208           PVE_PASSWORD="hunter3"
1209           PVE_VERIFY_SSL="false"
1210         '';
1211       };
1212     in {
1213       exporterConfig = {
1214         enable = true;
1215         environmentFile = pveExporterEnvFile;
1216       };
1217       exporterTest = ''
1218         wait_for_unit("prometheus-pve-exporter.service")
1219         wait_for_open_port(9221)
1220         wait_until_succeeds("curl localhost:9221")
1221       '';
1222     };
1223
1224     py-air-control = {
1225       nodeName = "py_air_control";
1226       exporterConfig = {
1227         enable = true;
1228         deviceHostname = "127.0.0.1";
1229       };
1230       exporterTest = ''
1231         wait_for_unit("prometheus-py-air-control-exporter.service")
1232         wait_for_open_port(9896)
1233         succeed(
1234             "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
1235         )
1236       '';
1237     };
1238
1239     redis = {
1240       exporterConfig = {
1241         enable = true;
1242       };
1243       metricProvider.services.redis.servers."".enable = true;
1244       exporterTest = ''
1245         wait_for_unit("redis.service")
1246         wait_for_unit("prometheus-redis-exporter.service")
1247         wait_for_open_port(6379)
1248         wait_for_open_port(9121)
1249         wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1250       '';
1251     };
1252
1253     restic =
1254       let
1255         repository = "rest:http://127.0.0.1:8000";
1256         passwordFile = pkgs.writeText "restic-test-password" "test-password";
1257       in
1258       {
1259         exporterConfig = {
1260           enable = true;
1261           inherit repository passwordFile;
1262         };
1263         metricProvider = {
1264           services.restic.server = {
1265             enable = true;
1266             extraFlags = [ "--no-auth" ];
1267           };
1268           environment.systemPackages = [ pkgs.restic ];
1269         };
1270         exporterTest = ''
1271           # prometheus-restic-exporter.service fails without initialised repository
1272           systemctl("stop prometheus-restic-exporter.service")
1273
1274           # Initialise the repository
1275           wait_for_unit("restic-rest-server.service")
1276           wait_for_open_port(8000)
1277           succeed("restic init --repo ${repository} --password-file ${passwordFile}")
1278
1279           systemctl("start prometheus-restic-exporter.service")
1280           wait_for_unit("prometheus-restic-exporter.service")
1281           wait_for_open_port(9753)
1282           wait_until_succeeds("curl -sSf localhost:9753/metrics | grep 'restic_check_success 1.0'")
1283         '';
1284       };
1285
1286     rspamd = {
1287       exporterConfig = {
1288         enable = true;
1289       };
1290       metricProvider = {
1291         services.rspamd.enable = true;
1292       };
1293       exporterTest = ''
1294         wait_for_unit("rspamd.service")
1295         wait_for_unit("prometheus-rspamd-exporter.service")
1296         wait_for_open_port(11334)
1297         wait_for_open_port(7980)
1298         wait_until_succeeds(
1299             "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1300         )
1301       '';
1302     };
1303
1304     rtl_433 = {
1305       exporterConfig = {
1306         enable = true;
1307       };
1308       metricProvider = {
1309         # Mock rtl_433 binary to return a dummy metric stream.
1310         nixpkgs.overlays = [
1311           (self: super: {
1312             rtl_433 = self.runCommand "rtl_433" { } ''
1313               mkdir -p "$out/bin"
1314               cat <<EOF > "$out/bin/rtl_433"
1315               #!/bin/sh
1316               while true; do
1317                 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1318                 sleep 4
1319               done
1320               EOF
1321               chmod +x "$out/bin/rtl_433"
1322             '';
1323           })
1324         ];
1325       };
1326       exporterTest = ''
1327         wait_for_unit("prometheus-rtl_433-exporter.service")
1328         wait_for_open_port(9550)
1329         wait_until_succeeds(
1330             "curl -sSf localhost:9550/metrics | grep '{}'".format(
1331                 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1332             )
1333         )
1334       '';
1335     };
1336
1337     sabnzbd = {
1338       exporterConfig = {
1339         enable = true;
1340         servers = [{
1341           baseUrl = "http://localhost:8080";
1342           apiKeyFile = "/var/sabnzbd-apikey";
1343         }];
1344       };
1345
1346       metricProvider = {
1347         services.sabnzbd.enable = true;
1348
1349         # unrar is required for sabnzbd
1350         nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "unrar" ];
1351
1352         # extract the generated api key before starting
1353         systemd.services.sabnzbd-apikey = {
1354           requires = [ "sabnzbd.service" ];
1355           after = [ "sabnzbd.service" ];
1356           requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
1357           before = [ "prometheus-sabnzbd-exporter.service" ];
1358           script = ''
1359             grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
1360           '';
1361         };
1362       };
1363
1364       exporterTest = ''
1365         wait_for_unit("sabnzbd.service")
1366         wait_for_unit("prometheus-sabnzbd-exporter.service")
1367         wait_for_open_port(8080)
1368         wait_for_open_port(9387)
1369         wait_until_succeeds(
1370             "curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
1371         )
1372       '';
1373     };
1374
1375     scaphandre = {
1376       exporterConfig = {
1377         enable = true;
1378       };
1379       metricProvider = {
1380         boot.kernelModules = [ "intel_rapl_common" ];
1381       };
1382       exporterTest = ''
1383         wait_for_unit("prometheus-scaphandre-exporter.service")
1384         wait_for_open_port(8080)
1385         wait_until_succeeds(
1386             "curl -sSf 'localhost:8080/metrics'"
1387         )
1388       '';
1389     };
1390
1391     shelly = {
1392       exporterConfig = {
1393         enable = true;
1394         metrics-file = "${pkgs.writeText "test.json" ''{}''}";
1395       };
1396       exporterTest = ''
1397         wait_for_unit("prometheus-shelly-exporter.service")
1398         wait_for_open_port(9784)
1399         wait_until_succeeds(
1400             "curl -sSf 'localhost:9784/metrics'"
1401         )
1402       '';
1403     };
1404
1405     script = {
1406       exporterConfig = {
1407         enable = true;
1408         settings.scripts = [
1409           { name = "success"; script = "sleep 1"; }
1410         ];
1411       };
1412       exporterTest = ''
1413         wait_for_unit("prometheus-script-exporter.service")
1414         wait_for_open_port(9172)
1415         wait_until_succeeds(
1416             "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1417                 'script_success{script="success"} 1'
1418             )
1419         )
1420       '';
1421     };
1422
1423     smartctl = {
1424       exporterConfig = {
1425         enable = true;
1426         devices = [
1427           "/dev/vda"
1428         ];
1429       };
1430       exporterTest = ''
1431         wait_until_succeeds(
1432             'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
1433         )
1434       '';
1435     };
1436
1437     smokeping = {
1438       exporterConfig = {
1439         enable = true;
1440         hosts = [ "127.0.0.1" ];
1441       };
1442       exporterTest = ''
1443         wait_for_unit("prometheus-smokeping-exporter.service")
1444         wait_for_open_port(9374)
1445         wait_until_succeeds(
1446             "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1447                 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source=""} '
1448             )
1449         )
1450         wait_until_succeeds(
1451             "curl -sSf localhost:9374/metrics | grep '{}'".format(
1452                 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source=""}'
1453             )
1454         )
1455       '';
1456     };
1457
1458     snmp = {
1459       exporterConfig = {
1460         enable = true;
1461         configuration = {
1462           auths.public_v2 = {
1463             community = "public";
1464             version = 2;
1465           };
1466         };
1467       };
1468       exporterTest = ''
1469         wait_for_unit("prometheus-snmp-exporter.service")
1470         wait_for_open_port(9116)
1471         succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1472       '';
1473     };
1474
1475     sql = {
1476       exporterConfig = {
1477         configuration.jobs.points = {
1478           interval = "1m";
1479           connections = [
1480             "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1481           ];
1482           queries = {
1483             points = {
1484               labels = [ "name" ];
1485               help = "Amount of points accumulated per person";
1486               values = [ "amount" ];
1487               query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1488             };
1489           };
1490         };
1491         enable = true;
1492         user = "prometheus-sql-exporter";
1493       };
1494       metricProvider = {
1495         services.postgresql = {
1496           enable = true;
1497           initialScript = builtins.toFile "init.sql" ''
1498             CREATE DATABASE data;
1499             \c data;
1500             CREATE TABLE points (amount INT, name TEXT);
1501             INSERT INTO points(amount, name) VALUES (1, 'jack');
1502             INSERT INTO points(amount, name) VALUES (2, 'jill');
1503             INSERT INTO points(amount, name) VALUES (3, 'jack');
1504
1505             CREATE USER "prometheus-sql-exporter";
1506             GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1507             GRANT SELECT ON points TO "prometheus-sql-exporter";
1508           '';
1509         };
1510         systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1511       };
1512       exporterTest = ''
1513         wait_for_unit("prometheus-sql-exporter.service")
1514         wait_for_open_port(9237)
1515         succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1516       '';
1517     };
1518
1519     statsd = {
1520       exporterConfig = {
1521         enable = true;
1522       };
1523       exporterTest = ''
1524         wait_for_unit("prometheus-statsd-exporter.service")
1525         wait_for_open_port(9102)
1526         succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
1527         wait_until_succeeds(
1528           "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
1529           curl http://localhost:9102/metrics | grep 'test_udp 1'",
1530           timeout=10
1531         )
1532         wait_until_succeeds(
1533           "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
1534           curl http://localhost:9102/metrics | grep 'test_tcp 1'",
1535           timeout=10
1536         )
1537       '';
1538     };
1539
1540     surfboard = {
1541       exporterConfig = {
1542         enable = true;
1543         modemAddress = "localhost";
1544       };
1545       metricProvider = {
1546         systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1547         services.nginx = {
1548           enable = true;
1549           virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1550             return 204;
1551           '';
1552         };
1553       };
1554       exporterTest = ''
1555         wait_for_unit("nginx.service")
1556         wait_for_open_port(80)
1557         wait_for_unit("prometheus-surfboard-exporter.service")
1558         wait_for_open_port(9239)
1559         succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1560       '';
1561     };
1562
1563     systemd = {
1564       exporterConfig = {
1565         enable = true;
1566
1567         extraFlags = [
1568           "--systemd.collector.enable-restart-count"
1569         ];
1570       };
1571       metricProvider = { };
1572       exporterTest = ''
1573         wait_for_unit("prometheus-systemd-exporter.service")
1574         wait_for_open_port(9558)
1575         wait_until_succeeds(
1576             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1577                 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1578             )
1579         )
1580         succeed(
1581             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1582                 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1583             )
1584         )
1585       '';
1586     };
1587
1588     unpoller = {
1589       nodeName = "unpoller";
1590       exporterConfig.enable = true;
1591       exporterConfig.controllers = [{ }];
1592       exporterTest = ''
1593         wait_until_succeeds(
1594             'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
1595         )
1596       '';
1597     };
1598
1599     unbound = {
1600       exporterConfig = {
1601         enable = true;
1602         unbound.host = "unix:///run/unbound/unbound.ctl";
1603       };
1604       metricProvider = {
1605         services.unbound = {
1606           enable = true;
1607           localControlSocketPath = "/run/unbound/unbound.ctl";
1608         };
1609         systemd.services.prometheus-unbound-exporter.serviceConfig = {
1610           SupplementaryGroups = [ "unbound" ];
1611         };
1612       };
1613       exporterTest = ''
1614         wait_for_unit("unbound.service")
1615         wait_for_unit("prometheus-unbound-exporter.service")
1616         wait_for_open_port(9167)
1617         wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1618       '';
1619     };
1620
1621     v2ray = {
1622       exporterConfig = {
1623         enable = true;
1624       };
1625
1626       metricProvider = {
1627         systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1628         services.v2ray = {
1629           enable = true;
1630           config = {
1631             stats = {};
1632             api = {
1633               tag = "api";
1634               services = [ "StatsService" ];
1635             };
1636             inbounds = [
1637               {
1638                 port = 1080;
1639                 listen = "127.0.0.1";
1640                 protocol = "http";
1641               }
1642               {
1643                 listen = "127.0.0.1";
1644                 port = 54321;
1645                 protocol = "dokodemo-door";
1646                 settings = { address = "127.0.0.1"; };
1647                 tag = "api";
1648               }
1649             ];
1650             outbounds = [
1651               {
1652                 protocol = "freedom";
1653               }
1654               {
1655                 protocol = "freedom";
1656                 settings = {};
1657                 tag = "api";
1658               }
1659             ];
1660             routing = {
1661               strategy = "rules";
1662               settings = {
1663                 rules = [
1664                   {
1665                     inboundTag = [ "api" ];
1666                     outboundTag = "api";
1667                     type = "field";
1668                   }
1669                 ];
1670               };
1671             };
1672           };
1673         };
1674       };
1675       exporterTest = ''
1676         wait_for_unit("prometheus-v2ray-exporter.service")
1677         wait_for_open_port(9299)
1678         succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1679       '';
1680     };
1681
1682     varnish = {
1683       exporterConfig = {
1684         enable = true;
1685         instance = "/run/varnish/varnish";
1686         group = "varnish";
1687       };
1688       metricProvider = {
1689         systemd.services.prometheus-varnish-exporter.after = [
1690           "varnish.service"
1691         ];
1692         services.varnish = {
1693           enable = true;
1694           stateDir = "/run/varnish/varnish";
1695           config = ''
1696             vcl 4.0;
1697             backend default {
1698               .host = "127.0.0.1";
1699               .port = "80";
1700             }
1701           '';
1702         };
1703       };
1704       exporterTest = ''
1705         wait_for_unit("prometheus-varnish-exporter.service")
1706         wait_for_open_port(6081)
1707         wait_for_open_port(9131)
1708         succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1709       '';
1710     };
1711
1712     wireguard = let
1713       snakeoil = import ./wireguard/snakeoil-keys.nix;
1714       publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
1715     in
1716       {
1717         exporterConfig.enable = true;
1718         metricProvider = {
1719           networking.wireguard.interfaces.wg0 = {
1720             ips = [ "10.23.42.1/32" "fc00::1/128" ];
1721             listenPort = 23542;
1722
1723             inherit (snakeoil.peer0) privateKey;
1724
1725             peers = singleton {
1726               allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1727
1728               inherit (snakeoil.peer1) publicKey;
1729             };
1730           };
1731           systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1732         };
1733         exporterTest = ''
1734           wait_for_unit("prometheus-wireguard-exporter.service")
1735           wait_for_open_port(9586)
1736           wait_until_succeeds(
1737               "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
1738           )
1739         '';
1740       };
1741
1742     zfs = {
1743       exporterConfig = {
1744         enable = true;
1745       };
1746       metricProvider = {
1747         boot.supportedFilesystems = [ "zfs" ];
1748         networking.hostId = "7327ded7";
1749       };
1750       exporterTest = ''
1751         wait_for_unit("prometheus-zfs-exporter.service")
1752         wait_for_unit("zfs.target")
1753         wait_for_open_port(9134)
1754         wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1755       '';
1756     };
1757   };
1758 in
1759 mapAttrs
1760   (exporter: testConfig: (makeTest (
1761     let
1762       nodeName = testConfig.nodeName or exporter;
1763
1764     in
1765     {
1766       name = "prometheus-${exporter}-exporter";
1767
1768       nodes.${nodeName} = mkMerge [{
1769         services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1770       } testConfig.metricProvider or { }];
1771
1772       testScript = ''
1773         ${nodeName}.start()
1774         ${concatStringsSep "\n" (map (line:
1775           if builtins.any (b: b) [
1776             (builtins.match "^[[:space:]]*$" line != null)
1777             (builtins.substring 0 1 line == "#")
1778             (builtins.substring 0 1 line == " ")
1779             (builtins.substring 0 1 line == ")")
1780           ]
1781           then line
1782           else "${nodeName}.${line}"
1783         ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1784         ${nodeName}.shutdown()
1785       '';
1786
1787       meta = with maintainers; {
1788         maintainers = [ willibutz ];
1789       };
1790     }
1791   )))
1792   exporterTests