nixos/tests/prometheus-exporters.nix

   1 { system ? builtins.currentSystem
   2 , config ? { }
   3 , pkgs ? import ../.. { inherit system config; }
   4 }:
   5
   6 let
   7   inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
   8   inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
   9     removeSuffix replaceStrings singleton splitString makeBinPath;
  10
  11   /*
  12     * The attrset `exporterTests` contains one attribute
  13     * for each exporter test. Each of these attributes
  14     * is expected to be an attrset containing:
  15     *
  16     *  `exporterConfig`:
  17     *    this attribute set contains config for the exporter itself
  18     *
  19     *  `exporterTest`
  20     *    this attribute set contains test instructions
  21     *
  22     *  `metricProvider` (optional)
  23     *    this attribute contains additional machine config
  24     *
  25     *  `nodeName` (optional)
  26     *    override an incompatible testnode name
  27     *
  28     *  Example:
  29     *    exporterTests.<exporterName> = {
  30     *      exporterConfig = {
  31     *        enable = true;
  32     *      };
  33     *      metricProvider = {
  34     *        services.<metricProvider>.enable = true;
  35     *      };
  36     *      exporterTest = ''
  37     *        wait_for_unit("prometheus-<exporterName>-exporter.service")
  38     *        wait_for_open_port(1234)
  39     *        succeed("curl -sSf 'localhost:1234/metrics'")
  40     *      '';
  41     *    };
  42     *
  43     *  # this would generate the following test config:
  44     *
  45     *    nodes.<exporterName> = {
  46     *      services.prometheus.<exporterName> = {
  47     *        enable = true;
  48     *      };
  49     *      services.<metricProvider>.enable = true;
  50     *    };
  51     *
  52     *    testScript = ''
  53     *      <exporterName>.start()
  54     *      <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
  55     *      <exporterName>.wait_for_open_port(1234)
  56     *      <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
  57     *      <exporterName>.shutdown()
  58     *    '';
  59   */
  60
  61   exporterTests = {
  62     apcupsd = {
  63       exporterConfig = {
  64         enable = true;
  65       };
  66       metricProvider = {
  67         services.apcupsd.enable = true;
  68       };
  69       exporterTest = ''
  70         wait_for_unit("apcupsd.service")
  71         wait_for_open_port(3551)
  72         wait_for_unit("prometheus-apcupsd-exporter.service")
  73         wait_for_open_port(9162)
  74         succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
  75       '';
  76     };
  77
  78     artifactory = {
  79       exporterConfig = {
  80         enable = true;
  81         artiUsername = "artifactory-username";
  82         artiPassword = "artifactory-password";
  83       };
  84       exporterTest = ''
  85         wait_for_unit("prometheus-artifactory-exporter.service")
  86         wait_for_open_port(9531)
  87         succeed(
  88             "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
  89         )
  90       '';
  91     };
  92
  93     bind = {
  94       exporterConfig = {
  95         enable = true;
  96       };
  97       metricProvider = {
  98         services.bind.enable = true;
  99         services.bind.extraConfig = ''
 100           statistics-channels {
 101             inet 127.0.0.1 port 8053 allow { localhost; };
 102           };
 103         '';
 104       };
 105       exporterTest = ''
 106         wait_for_unit("prometheus-bind-exporter.service")
 107         wait_for_open_port(9119)
 108         succeed(
 109             "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
 110         )
 111       '';
 112     };
 113
 114     bird = {
 115       exporterConfig = {
 116         enable = true;
 117       };
 118       metricProvider = {
 119         services.bird2.enable = true;
 120         services.bird2.config = ''
 121           router id 127.0.0.1;
 122
 123           protocol kernel MyObviousTestString {
 124             ipv4 {
 125               import all;
 126               export none;
 127             };
 128           }
 129
 130           protocol device {
 131           }
 132         '';
 133       };
 134       exporterTest = ''
 135         wait_for_unit("prometheus-bird-exporter.service")
 136         wait_for_open_port(9324)
 137         wait_until_succeeds(
 138             "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
 139         )
 140       '';
 141     };
 142
 143     bitcoin = {
 144       exporterConfig = {
 145         enable = true;
 146         rpcUser = "bitcoinrpc";
 147         rpcPasswordFile = pkgs.writeText "password" "hunter2";
 148       };
 149       metricProvider = {
 150         services.bitcoind.default.enable = true;
 151         services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
 152       };
 153       exporterTest = ''
 154         wait_for_unit("prometheus-bitcoin-exporter.service")
 155         wait_for_unit("bitcoind-default.service")
 156         wait_for_open_port(9332)
 157         succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
 158       '';
 159     };
 160
 161     blackbox = {
 162       exporterConfig = {
 163         enable = true;
 164         configFile = pkgs.writeText "config.yml" (builtins.toJSON {
 165           modules.icmp_v6 = {
 166             prober = "icmp";
 167             icmp.preferred_ip_protocol = "ip6";
 168           };
 169         });
 170       };
 171       exporterTest = ''
 172         wait_for_unit("prometheus-blackbox-exporter.service")
 173         wait_for_open_port(9115)
 174         succeed(
 175             "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
 176         )
 177       '';
 178     };
 179
 180     collectd = {
 181       exporterConfig = {
 182         enable = true;
 183         extraFlags = [ "--web.collectd-push-path /collectd" ];
 184       };
 185       exporterTest = let postData = replaceStrings [ "\n" ] [ "" ] ''
 186         [{
 187           "values":[23],
 188           "dstypes":["gauge"],
 189           "type":"gauge",
 190           "interval":1000,
 191           "host":"testhost",
 192           "plugin":"testplugin",
 193           "time":DATE
 194         }]
 195       ''; in
 196         ''
 197           wait_for_unit("prometheus-collectd-exporter.service")
 198           wait_for_open_port(9103)
 199           succeed(
 200               'echo \'${postData}\'> /tmp/data.json'
 201           )
 202           succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
 203           succeed(
 204               "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
 205           )
 206           succeed(
 207               "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
 208           )
 209         '';
 210     };
 211
 212     dnsmasq = {
 213       exporterConfig = {
 214         enable = true;
 215         leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
 216       };
 217       metricProvider = {
 218         services.dnsmasq.enable = true;
 219       };
 220       exporterTest = ''
 221         wait_for_unit("prometheus-dnsmasq-exporter.service")
 222         wait_for_open_port(9153)
 223         succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
 224       '';
 225     };
 226
 227     # Access to WHOIS server is required to properly test this exporter, so
 228     # just perform basic sanity check that the exporter is running and returns
 229     # a failure.
 230     domain = {
 231       exporterConfig = {
 232         enable = true;
 233       };
 234       exporterTest = ''
 235         wait_for_unit("prometheus-domain-exporter.service")
 236         wait_for_open_port(9222)
 237         succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
 238       '';
 239     };
 240
 241     dovecot = {
 242       exporterConfig = {
 243         enable = true;
 244         scopes = [ "global" ];
 245         socketPath = "/var/run/dovecot2/old-stats";
 246         user = "root"; # <- don't use user root in production
 247       };
 248       metricProvider = {
 249         services.dovecot2.enable = true;
 250       };
 251       exporterTest = ''
 252         wait_for_unit("prometheus-dovecot-exporter.service")
 253         wait_for_open_port(9166)
 254         succeed(
 255             "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
 256         )
 257       '';
 258     };
 259
 260     exportarr-sonarr = {
 261       nodeName = "exportarr_sonarr";
 262       exporterConfig = {
 263         enable = true;
 264         url = "http://127.0.0.1:8989";
 265         # testing for real data is tricky, because the api key can not be preconfigured
 266         apiKeyFile = pkgs.writeText "dummy-api-key" "eccff6a992bc2e4b88e46d064b26bb4e";
 267       };
 268       exporterTest = ''
 269         wait_for_unit("prometheus-exportarr-sonarr-exporter.service")
 270         wait_for_open_port(9707)
 271         succeed("curl -sSf 'http://localhost:9707/metrics")
 272       '';
 273     };
 274
 275     fastly = {
 276       exporterConfig = {
 277         enable = true;
 278         tokenPath = pkgs.writeText "token" "abc123";
 279       };
 280
 281       # noop: fastly's exporter can't start without first talking to fastly
 282       # see: https://github.com/peterbourgon/fastly-exporter/issues/87
 283       exporterTest = ''
 284         succeed("true");
 285       '';
 286     };
 287
 288     fritzbox = {
 289       # TODO add proper test case
 290       exporterConfig = {
 291         enable = true;
 292       };
 293       exporterTest = ''
 294         wait_for_unit("prometheus-fritzbox-exporter.service")
 295         wait_for_open_port(9133)
 296         succeed(
 297             "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
 298         )
 299       '';
 300     };
 301
 302     graphite = {
 303       exporterConfig = {
 304         enable = true;
 305         port = 9108;
 306         graphitePort = 9109;
 307         mappingSettings.mappings = [{
 308           match = "test.*.*";
 309           name = "testing";
 310           labels = {
 311             protocol = "$1";
 312             author = "$2";
 313           };
 314         }];
 315       };
 316       exporterTest = ''
 317         wait_for_unit("prometheus-graphite-exporter.service")
 318         wait_for_open_port(9108)
 319         wait_for_open_port(9109)
 320         succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
 321         succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
 322       '';
 323     };
 324
 325     idrac = {
 326       exporterConfig = {
 327         enable = true;
 328         port = 9348;
 329         configuration = {
 330           hosts = {
 331             default = { username = "username"; password = "password"; };
 332           };
 333         };
 334       };
 335       exporterTest = ''
 336         wait_for_unit("prometheus-idrac-exporter.service")
 337         wait_for_open_port(9348)
 338         wait_until_succeeds("curl localhost:9348")
 339       '';
 340     };
 341
 342     influxdb = {
 343       exporterConfig = {
 344         enable = true;
 345         sampleExpiry = "3s";
 346       };
 347       exporterTest = ''
 348         wait_for_unit("prometheus-influxdb-exporter.service")
 349         wait_for_open_port(9122)
 350         succeed(
 351           "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
 352         )
 353         succeed(
 354           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 355         )
 356         execute("sleep 5")
 357         fail(
 358           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 359         )
 360       '';
 361     };
 362
 363     ipmi = {
 364       exporterConfig = {
 365         enable = true;
 366       };
 367       exporterTest = ''
 368         wait_for_unit("prometheus-ipmi-exporter.service")
 369         wait_for_open_port(9290)
 370         succeed(
 371           "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
 372         )
 373       '';
 374     };
 375
 376     jitsi = {
 377       exporterConfig = {
 378         enable = true;
 379       };
 380       metricProvider = {
 381         systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
 382         services.jitsi-videobridge = {
 383           enable = true;
 384           colibriRestApi = true;
 385         };
 386       };
 387       exporterTest = ''
 388         wait_for_unit("jitsi-videobridge2.service")
 389         wait_for_open_port(8080)
 390         wait_for_unit("prometheus-jitsi-exporter.service")
 391         wait_for_open_port(9700)
 392         wait_until_succeeds(
 393             'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
 394         )
 395         succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
 396       '';
 397     };
 398
 399     json = {
 400       exporterConfig = {
 401         enable = true;
 402         url = "http://localhost";
 403         configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
 404           modules = {
 405             default = {
 406               metrics = [
 407                 { name = "json_test_metric"; path = "{ .test }"; }
 408               ];
 409             };
 410           };
 411         });
 412       };
 413       metricProvider = {
 414         systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
 415         services.nginx = {
 416           enable = true;
 417           virtualHosts.localhost.locations."/".extraConfig = ''
 418             return 200 "{\"test\":1}";
 419           '';
 420         };
 421       };
 422       exporterTest = ''
 423         wait_for_unit("nginx.service")
 424         wait_for_open_port(80)
 425         wait_for_unit("prometheus-json-exporter.service")
 426         wait_for_open_port(7979)
 427         succeed(
 428             "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
 429         )
 430       '';
 431     };
 432
 433     kea = let
 434       controlSocketPathV4 = "/run/kea-dhcp4/dhcp4.sock";
 435       controlSocketPathV6 = "/run/kea-dhcp6/dhcp6.sock";
 436     in
 437     {
 438       exporterConfig = {
 439         enable = true;
 440         controlSocketPaths = [
 441           controlSocketPathV4
 442           controlSocketPathV6
 443         ];
 444       };
 445       metricProvider = {
 446         services.kea = {
 447           dhcp4 = {
 448             enable = true;
 449             settings = {
 450               control-socket = {
 451                 socket-type = "unix";
 452                 socket-name = controlSocketPathV4;
 453               };
 454             };
 455           };
 456           dhcp6 = {
 457             enable = true;
 458             settings = {
 459               control-socket = {
 460                 socket-type = "unix";
 461                 socket-name = controlSocketPathV6;
 462               };
 463             };
 464           };
 465         };
 466       };
 467
 468       exporterTest = ''
 469         wait_for_unit("kea-dhcp4-server.service")
 470         wait_for_unit("kea-dhcp6-server.service")
 471         wait_for_file("${controlSocketPathV4}")
 472         wait_for_file("${controlSocketPathV6}")
 473         wait_for_unit("prometheus-kea-exporter.service")
 474         wait_for_open_port(9547)
 475         succeed(
 476             "curl --fail localhost:9547/metrics | grep 'packets_received_total'"
 477         )
 478       '';
 479     };
 480
 481     knot = {
 482       exporterConfig = {
 483         enable = true;
 484       };
 485       metricProvider = {
 486         services.knot = {
 487           enable = true;
 488           extraArgs = [ "-v" ];
 489           settingsFile = pkgs.writeText "knot.conf" ''
 490             server:
 491               listen: 127.0.0.1@53
 492
 493             template:
 494               - id: default
 495                 global-module: mod-stats
 496                 dnssec-signing: off
 497                 zonefile-sync: -1
 498                 journal-db: /var/lib/knot/journal
 499                 kasp-db: /var/lib/knot/kasp
 500                 timer-db: /var/lib/knot/timer
 501                 zonefile-load: difference
 502                 storage: ${pkgs.buildEnv {
 503                   name = "foo";
 504                   paths = [
 505                     (pkgs.writeTextDir "test.zone" ''
 506                       @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
 507                       @       NS      ns1
 508                       @       NS      ns2
 509                       ns1     A       192.168.0.1
 510                     '')
 511                   ];
 512                 }}
 513
 514             mod-stats:
 515               - id: custom
 516                 edns-presence: on
 517                 query-type: on
 518
 519             zone:
 520               - domain: test
 521                 file: test.zone
 522                 module: mod-stats/custom
 523           '';
 524         };
 525       };
 526       exporterTest = ''
 527         wait_for_unit("knot.service")
 528         wait_for_unit("prometheus-knot-exporter.service")
 529         wait_for_open_port(9433)
 530         succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
 531       '';
 532     };
 533
 534     keylight = {
 535       # A hardware device is required to properly test this exporter, so just
 536       # perform a couple of basic sanity checks that the exporter is running
 537       # and requires a target, but cannot reach a specified target.
 538       exporterConfig = {
 539         enable = true;
 540       };
 541       exporterTest = ''
 542         wait_for_unit("prometheus-keylight-exporter.service")
 543         wait_for_open_port(9288)
 544         succeed(
 545             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
 546         )
 547         succeed(
 548             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
 549         )
 550       '';
 551     };
 552
 553     lnd = {
 554       exporterConfig = {
 555         enable = true;
 556         lndTlsPath = "/var/lib/lnd/tls.cert";
 557         lndMacaroonDir = "/var/lib/lnd";
 558         extraFlags = [ "--lnd.network=regtest" ];
 559       };
 560       metricProvider = {
 561         systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
 562         systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
 563         services.bitcoind.regtest = {
 564           enable = true;
 565           extraConfig = ''
 566             rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
 567             zmqpubrawblock=tcp://127.0.0.1:28332
 568             zmqpubrawtx=tcp://127.0.0.1:28333
 569           '';
 570           extraCmdlineOptions = [ "-regtest" ];
 571         };
 572         systemd.services.lnd = {
 573           serviceConfig.ExecStart = ''
 574             ${pkgs.lnd}/bin/lnd \
 575               --datadir=/var/lib/lnd \
 576               --tlscertpath=/var/lib/lnd/tls.cert \
 577               --tlskeypath=/var/lib/lnd/tls.key \
 578               --logdir=/var/log/lnd \
 579               --bitcoin.active \
 580               --bitcoin.regtest \
 581               --bitcoin.node=bitcoind \
 582               --bitcoind.rpcuser=bitcoinrpc \
 583               --bitcoind.rpcpass=hunter2 \
 584               --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
 585               --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
 586               --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
 587           '';
 588           serviceConfig.StateDirectory = "lnd";
 589           wantedBy = [ "multi-user.target" ];
 590           after = [ "network.target" ];
 591         };
 592         # initialize wallet, creates macaroon needed by exporter
 593         systemd.services.lnd.postStart = ''
 594           ${pkgs.curl}/bin/curl \
 595             --retry 20 \
 596             --retry-delay 1 \
 597             --retry-connrefused \
 598             --cacert /var/lib/lnd/tls.cert \
 599             -X GET \
 600             https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
 601           ${pkgs.curl}/bin/curl \
 602             --retry 20 \
 603             --retry-delay 1 \
 604             --retry-connrefused \
 605             --cacert /var/lib/lnd/tls.cert \
 606             -X POST \
 607             -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
 608             https://localhost:8080/v1/initwallet
 609         '';
 610       };
 611       exporterTest = ''
 612         wait_for_unit("lnd.service")
 613         wait_for_open_port(10009)
 614         wait_for_unit("prometheus-lnd-exporter.service")
 615         wait_for_open_port(9092)
 616         succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
 617       '';
 618     };
 619
 620     mail = {
 621       exporterConfig = {
 622         enable = true;
 623         configuration = {
 624           monitoringInterval = "2s";
 625           mailCheckTimeout = "10s";
 626           servers = [{
 627             name = "testserver";
 628             server = "localhost";
 629             port = 25;
 630             from = "mail-exporter@localhost";
 631             to = "mail-exporter@localhost";
 632             detectionDir = "/var/spool/mail/mail-exporter/new";
 633           }];
 634         };
 635       };
 636       metricProvider = {
 637         services.postfix.enable = true;
 638         systemd.services.prometheus-mail-exporter = {
 639           after = [ "postfix.service" ];
 640           requires = [ "postfix.service" ];
 641           serviceConfig = {
 642             ExecStartPre = [
 643               "${pkgs.writeShellScript "create-maildir" ''
 644                 mkdir -p -m 0700 mail-exporter/new
 645               ''}"
 646             ];
 647             ProtectHome = true;
 648             ReadOnlyPaths = "/";
 649             ReadWritePaths = "/var/spool/mail";
 650             WorkingDirectory = "/var/spool/mail";
 651           };
 652         };
 653         users.users.mailexporter = {
 654           isSystemUser = true;
 655           group = "mailexporter";
 656         };
 657         users.groups.mailexporter = {};
 658       };
 659       exporterTest = ''
 660         wait_for_unit("postfix.service")
 661         wait_for_unit("prometheus-mail-exporter.service")
 662         wait_for_open_port(9225)
 663         wait_until_succeeds(
 664             "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
 665         )
 666       '';
 667     };
 668
 669     mikrotik = {
 670       exporterConfig = {
 671         enable = true;
 672         extraFlags = [ "-timeout=1s" ];
 673         configuration = {
 674           devices = [
 675             {
 676               name = "router";
 677               address = "192.168.42.48";
 678               user = "prometheus";
 679               password = "shh";
 680             }
 681           ];
 682           features = {
 683             bgp = true;
 684             dhcp = true;
 685             dhcpl = true;
 686             dhcpv6 = true;
 687             health = true;
 688             routes = true;
 689             poe = true;
 690             pools = true;
 691             optics = true;
 692             w60g = true;
 693             wlansta = true;
 694             wlanif = true;
 695             monitor = true;
 696             ipsec = true;
 697           };
 698         };
 699       };
 700       exporterTest = ''
 701         wait_for_unit("prometheus-mikrotik-exporter.service")
 702         wait_for_open_port(9436)
 703         succeed(
 704             "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
 705         )
 706       '';
 707     };
 708
 709     modemmanager = {
 710       exporterConfig = {
 711         enable = true;
 712         refreshRate = "10s";
 713       };
 714       metricProvider = {
 715         # ModemManager is installed when NetworkManager is enabled. Ensure it is
 716         # started and is wanted by NM and the exporter to start everything up
 717         # in the right order.
 718         networking.networkmanager.enable = true;
 719         systemd.services.ModemManager = {
 720           enable = true;
 721           wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
 722         };
 723       };
 724       exporterTest = ''
 725         wait_for_unit("ModemManager.service")
 726         wait_for_unit("prometheus-modemmanager-exporter.service")
 727         wait_for_open_port(9539)
 728         succeed(
 729             "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
 730         )
 731       '';
 732     };
 733
 734     mysqld = {
 735       exporterConfig = {
 736         enable = true;
 737         runAsLocalSuperUser = true;
 738         configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
 739           [client]
 740           user = exporter
 741           password = snakeoilpassword
 742         '';
 743       };
 744       metricProvider = {
 745         services.mysql = {
 746           enable = true;
 747           package = pkgs.mariadb;
 748           initialScript = pkgs.writeText "mysql-init-script.sql" ''
 749             CREATE USER 'exporter'@'localhost'
 750             IDENTIFIED BY 'snakeoilpassword'
 751             WITH MAX_USER_CONNECTIONS 3;
 752             GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
 753           '';
 754         };
 755       };
 756       exporterTest = ''
 757         wait_for_unit("prometheus-mysqld-exporter.service")
 758         wait_for_open_port(9104)
 759         wait_for_unit("mysql.service")
 760         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 761         systemctl("stop mysql.service")
 762         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
 763         systemctl("start mysql.service")
 764         wait_for_unit("mysql.service")
 765         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 766       '';
 767     };
 768
 769     nextcloud = {
 770       exporterConfig = {
 771         enable = true;
 772         passwordFile = "/var/nextcloud-pwfile";
 773         url = "http://localhost";
 774       };
 775       metricProvider = {
 776         systemd.services.nc-pwfile =
 777           let
 778             passfile = (pkgs.writeText "pwfile" "snakeoilpw");
 779           in
 780           {
 781             requiredBy = [ "prometheus-nextcloud-exporter.service" ];
 782             before = [ "prometheus-nextcloud-exporter.service" ];
 783             serviceConfig.ExecStart = ''
 784               ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
 785             '';
 786           };
 787         services.nginx = {
 788           enable = true;
 789           virtualHosts."localhost" = {
 790             basicAuth.nextcloud-exporter = "snakeoilpw";
 791             locations."/" = {
 792               root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
 793               tryFiles = "/negative-space.json =404";
 794             };
 795           };
 796         };
 797       };
 798       exporterTest = ''
 799         wait_for_unit("nginx.service")
 800         wait_for_unit("prometheus-nextcloud-exporter.service")
 801         wait_for_open_port(9205)
 802         succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
 803       '';
 804     };
 805
 806     nginx = {
 807       exporterConfig = {
 808         enable = true;
 809       };
 810       metricProvider = {
 811         services.nginx = {
 812           enable = true;
 813           statusPage = true;
 814           virtualHosts."test".extraConfig = "return 204;";
 815         };
 816       };
 817       exporterTest = ''
 818         wait_for_unit("nginx.service")
 819         wait_for_unit("prometheus-nginx-exporter.service")
 820         wait_for_open_port(9113)
 821         succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up 1'")
 822       '';
 823     };
 824
 825     nginxlog = {
 826       exporterConfig = {
 827         enable = true;
 828         group = "nginx";
 829         settings = {
 830           namespaces = [
 831             {
 832               name = "filelogger";
 833               source = {
 834                 files = [ "/var/log/nginx/filelogger.access.log" ];
 835               };
 836             }
 837             {
 838               name = "syslogger";
 839               source = {
 840                 syslog = {
 841                   listen_address = "udp://127.0.0.1:10000";
 842                   format = "rfc3164";
 843                   tags = [ "nginx" ];
 844                 };
 845               };
 846             }
 847           ];
 848         };
 849       };
 850       metricProvider = {
 851         services.nginx = {
 852           enable = true;
 853           httpConfig = ''
 854             server {
 855               listen 80;
 856               server_name filelogger.local;
 857               access_log /var/log/nginx/filelogger.access.log;
 858             }
 859             server {
 860               listen 81;
 861               server_name syslogger.local;
 862               access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
 863             }
 864           '';
 865         };
 866       };
 867       exporterTest = ''
 868         wait_for_unit("nginx.service")
 869         wait_for_unit("prometheus-nginxlog-exporter.service")
 870         wait_for_open_port(9117)
 871         wait_for_open_port(80)
 872         wait_for_open_port(81)
 873         succeed("curl http://localhost")
 874         execute("sleep 1")
 875         succeed(
 876             "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
 877         )
 878         succeed("curl http://localhost:81")
 879         execute("sleep 1")
 880         succeed(
 881             "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
 882         )
 883       '';
 884     };
 885
 886     node = {
 887       exporterConfig = {
 888         enable = true;
 889       };
 890       exporterTest = ''
 891         wait_for_unit("prometheus-node-exporter.service")
 892         wait_for_open_port(9100)
 893         succeed(
 894             "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
 895         )
 896       '';
 897     };
 898
 899     openldap = {
 900       exporterConfig = {
 901         enable = true;
 902         ldapCredentialFile = "${pkgs.writeText "exporter.yml" ''
 903           ldapUser: "cn=root,dc=example"
 904           ldapPass: "notapassword"
 905         ''}";
 906       };
 907       metricProvider = {
 908         services.openldap = {
 909           enable = true;
 910           settings.children = {
 911             "cn=schema".includes = [
 912               "${pkgs.openldap}/etc/schema/core.ldif"
 913               "${pkgs.openldap}/etc/schema/cosine.ldif"
 914               "${pkgs.openldap}/etc/schema/inetorgperson.ldif"
 915               "${pkgs.openldap}/etc/schema/nis.ldif"
 916             ];
 917             "olcDatabase={1}mdb" = {
 918               attrs = {
 919                 objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
 920                 olcDatabase = "{1}mdb";
 921                 olcDbDirectory = "/var/db/openldap";
 922                 olcSuffix = "dc=example";
 923                 olcRootDN = {
 924                   # cn=root,dc=example
 925                   base64 = "Y249cm9vdCxkYz1leGFtcGxl";
 926                 };
 927                 olcRootPW = {
 928                   path = "${pkgs.writeText "rootpw" "notapassword"}";
 929                 };
 930               };
 931             };
 932             "olcDatabase={2}monitor".attrs = {
 933               objectClass = [ "olcDatabaseConfig" ];
 934               olcDatabase = "{2}monitor";
 935               olcAccess = [ "to dn.subtree=cn=monitor by users read" ];
 936             };
 937           };
 938           declarativeContents."dc=example" = ''
 939             dn: dc=example
 940             objectClass: domain
 941             dc: example
 942
 943             dn: ou=users,dc=example
 944             objectClass: organizationalUnit
 945             ou: users
 946           '';
 947         };
 948       };
 949       exporterTest = ''
 950         wait_for_unit("prometheus-openldap-exporter.service")
 951         wait_for_open_port(389)
 952         wait_for_open_port(9330)
 953         wait_until_succeeds(
 954             "curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'"
 955         )
 956       '';
 957     };
 958
 959     openvpn = {
 960       exporterConfig = {
 961         enable = true;
 962         group = "openvpn";
 963         statusPaths = [ "/run/openvpn-test" ];
 964       };
 965       metricProvider = {
 966         users.groups.openvpn = { };
 967         services.openvpn.servers.test = {
 968           config = ''
 969             dev tun
 970             status /run/openvpn-test
 971             status-version 3
 972           '';
 973           up = "chmod g+r /run/openvpn-test";
 974         };
 975         systemd.services."openvpn-test".serviceConfig.Group = "openvpn";
 976       };
 977       exporterTest = ''
 978         wait_for_unit("openvpn-test.service")
 979         wait_for_unit("prometheus-openvpn-exporter.service")
 980         succeed("curl -sSf http://localhost:9176/metrics | grep 'openvpn_up{.*} 1'")
 981       '';
 982     };
 983
 984     pgbouncer = {
 985       exporterConfig = {
 986         enable = true;
 987         connectionStringFile = pkgs.writeText "connection.conf" "postgres://admin:@localhost:6432/pgbouncer?sslmode=disable";
 988       };
 989
 990       metricProvider = {
 991         services.postgresql.enable = true;
 992         services.pgbouncer = {
 993           # https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
 994           ignoreStartupParameters = "extra_float_digits";
 995           enable = true;
 996           listenAddress = "*";
 997           databases = { postgres = "host=/run/postgresql/ port=5432 auth_user=postgres dbname=postgres"; };
 998           authType = "any";
 999           maxClientConn = 99;
1000         };
1001       };
1002       exporterTest = ''
1003         wait_for_unit("postgresql.service")
1004         wait_for_unit("pgbouncer.service")
1005         wait_for_unit("prometheus-pgbouncer-exporter.service")
1006         wait_for_open_port(9127)
1007         succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
1008         succeed(
1009             "curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
1010         )
1011       '';
1012     };
1013
1014     php-fpm = {
1015       nodeName = "php_fpm";
1016       exporterConfig = {
1017         enable = true;
1018         environmentFile = pkgs.writeTextFile {
1019           name = "/tmp/prometheus-php-fpm-exporter.env";
1020           text = ''
1021             PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
1022           '';
1023         };
1024       };
1025       metricProvider = {
1026         users.users."php-fpm-exporter" = {
1027           isSystemUser = true;
1028           group  = "php-fpm-exporter";
1029         };
1030         users.groups."php-fpm-exporter" = {};
1031         services.phpfpm.pools."php-fpm-exporter" = {
1032           user = "php-fpm-exporter";
1033           group = "php-fpm-exporter";
1034           settings = {
1035             "pm" = "dynamic";
1036             "pm.max_children" = 32;
1037             "pm.max_requests" = 500;
1038             "pm.start_servers" = 2;
1039             "pm.min_spare_servers" = 2;
1040             "pm.max_spare_servers" = 5;
1041             "pm.status_path" = "/status";
1042             "listen" = "127.0.0.1:9000";
1043             "listen.allowed_clients" = "127.0.0.1";
1044           };
1045           phpEnv."PATH" = makeBinPath [ pkgs.php ];
1046         };
1047       };
1048       exporterTest = ''
1049         wait_for_unit("phpfpm-php-fpm-exporter.service")
1050         wait_for_unit("prometheus-php-fpm-exporter.service")
1051         succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
1052       '';
1053     };
1054
1055     postfix = {
1056       exporterConfig = {
1057         enable = true;
1058       };
1059       metricProvider = {
1060         services.postfix.enable = true;
1061       };
1062       exporterTest = ''
1063         wait_for_unit("prometheus-postfix-exporter.service")
1064         wait_for_file("/var/lib/postfix/queue/public/showq")
1065         wait_for_open_port(9154)
1066         wait_until_succeeds(
1067             "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
1068         )
1069         succeed(
1070             "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
1071         )
1072         succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
1073       '';
1074     };
1075
1076     postgres = {
1077       exporterConfig = {
1078         enable = true;
1079         runAsLocalSuperUser = true;
1080       };
1081       metricProvider = {
1082         services.postgresql.enable = true;
1083       };
1084       exporterTest = ''
1085         wait_for_unit("prometheus-postgres-exporter.service")
1086         wait_for_open_port(9187)
1087         wait_for_unit("postgresql.service")
1088         succeed(
1089             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1090         )
1091         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1092         systemctl("stop postgresql.service")
1093         succeed(
1094             "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
1095         )
1096         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
1097         systemctl("start postgresql.service")
1098         wait_for_unit("postgresql.service")
1099         succeed(
1100             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1101         )
1102         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1103       '';
1104     };
1105
1106     process = {
1107       exporterConfig = {
1108         enable = true;
1109         settings.process_names = [
1110           # Remove nix store path from process name
1111           { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
1112         ];
1113       };
1114       exporterTest = ''
1115         wait_for_unit("prometheus-process-exporter.service")
1116         wait_for_open_port(9256)
1117         wait_until_succeeds(
1118             "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
1119                 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
1120             )
1121         )
1122       '';
1123     };
1124
1125     pve = let
1126       pveExporterEnvFile = pkgs.writeTextFile {
1127         name = "pve.env";
1128         text = ''
1129           PVE_USER="test_user@pam"
1130           PVE_PASSWORD="hunter3"
1131           PVE_VERIFY_SSL="false"
1132         '';
1133       };
1134     in {
1135       exporterConfig = {
1136         enable = true;
1137         environmentFile = pveExporterEnvFile;
1138       };
1139       exporterTest = ''
1140         wait_for_unit("prometheus-pve-exporter.service")
1141         wait_for_open_port(9221)
1142         wait_until_succeeds("curl localhost:9221")
1143       '';
1144     };
1145
1146     py-air-control = {
1147       nodeName = "py_air_control";
1148       exporterConfig = {
1149         enable = true;
1150         deviceHostname = "127.0.0.1";
1151       };
1152       exporterTest = ''
1153         wait_for_unit("prometheus-py-air-control-exporter.service")
1154         wait_for_open_port(9896)
1155         succeed(
1156             "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
1157         )
1158       '';
1159     };
1160
1161     redis = {
1162       exporterConfig = {
1163         enable = true;
1164       };
1165       metricProvider.services.redis.servers."".enable = true;
1166       exporterTest = ''
1167         wait_for_unit("redis.service")
1168         wait_for_unit("prometheus-redis-exporter.service")
1169         wait_for_open_port(6379)
1170         wait_for_open_port(9121)
1171         wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1172       '';
1173     };
1174
1175     rspamd = {
1176       exporterConfig = {
1177         enable = true;
1178       };
1179       metricProvider = {
1180         services.rspamd.enable = true;
1181       };
1182       exporterTest = ''
1183         wait_for_unit("rspamd.service")
1184         wait_for_unit("prometheus-rspamd-exporter.service")
1185         wait_for_open_port(11334)
1186         wait_for_open_port(7980)
1187         wait_until_succeeds(
1188             "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1189         )
1190       '';
1191     };
1192
1193     rtl_433 = {
1194       exporterConfig = {
1195         enable = true;
1196       };
1197       metricProvider = {
1198         # Mock rtl_433 binary to return a dummy metric stream.
1199         nixpkgs.overlays = [
1200           (self: super: {
1201             rtl_433 = self.runCommand "rtl_433" { } ''
1202               mkdir -p "$out/bin"
1203               cat <<EOF > "$out/bin/rtl_433"
1204               #!/bin/sh
1205               while true; do
1206                 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1207                 sleep 4
1208               done
1209               EOF
1210               chmod +x "$out/bin/rtl_433"
1211             '';
1212           })
1213         ];
1214       };
1215       exporterTest = ''
1216         wait_for_unit("prometheus-rtl_433-exporter.service")
1217         wait_for_open_port(9550)
1218         wait_until_succeeds(
1219             "curl -sSf localhost:9550/metrics | grep '{}'".format(
1220                 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1221             )
1222         )
1223       '';
1224     };
1225
1226     sabnzbd = {
1227       exporterConfig = {
1228         enable = true;
1229         servers = [{
1230           baseUrl = "http://localhost:8080";
1231           apiKeyFile = "/var/sabnzbd-apikey";
1232         }];
1233       };
1234
1235       metricProvider = {
1236         services.sabnzbd.enable = true;
1237
1238         # unrar is required for sabnzbd
1239         nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "unrar" ];
1240
1241         # extract the generated api key before starting
1242         systemd.services.sabnzbd-apikey = {
1243           requires = [ "sabnzbd.service" ];
1244           after = [ "sabnzbd.service" ];
1245           requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
1246           before = [ "prometheus-sabnzbd-exporter.service" ];
1247           script = ''
1248             grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
1249           '';
1250         };
1251       };
1252
1253       exporterTest = ''
1254         wait_for_unit("sabnzbd.service")
1255         wait_for_unit("prometheus-sabnzbd-exporter.service")
1256         wait_for_open_port(8080)
1257         wait_for_open_port(9387)
1258         wait_until_succeeds(
1259             "curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
1260         )
1261       '';
1262     };
1263
1264     scaphandre = {
1265       exporterConfig = {
1266         enable = true;
1267       };
1268       metricProvider = {
1269         boot.kernelModules = [ "intel_rapl_common" ];
1270       };
1271       exporterTest = ''
1272         wait_for_unit("prometheus-scaphandre-exporter.service")
1273         wait_for_open_port(8080)
1274         wait_until_succeeds(
1275             "curl -sSf 'localhost:8080/metrics'"
1276         )
1277       '';
1278     };
1279
1280     shelly = {
1281       exporterConfig = {
1282         enable = true;
1283         metrics-file = "${pkgs.writeText "test.json" ''{}''}";
1284       };
1285       exporterTest = ''
1286         wait_for_unit("prometheus-shelly-exporter.service")
1287         wait_for_open_port(9784)
1288         wait_until_succeeds(
1289             "curl -sSf 'localhost:9784/metrics'"
1290         )
1291       '';
1292     };
1293
1294     script = {
1295       exporterConfig = {
1296         enable = true;
1297         settings.scripts = [
1298           { name = "success"; script = "sleep 1"; }
1299         ];
1300       };
1301       exporterTest = ''
1302         wait_for_unit("prometheus-script-exporter.service")
1303         wait_for_open_port(9172)
1304         wait_until_succeeds(
1305             "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1306                 'script_success{script="success"} 1'
1307             )
1308         )
1309       '';
1310     };
1311
1312     smartctl = {
1313       exporterConfig = {
1314         enable = true;
1315         devices = [
1316           "/dev/vda"
1317         ];
1318       };
1319       exporterTest = ''
1320         wait_until_succeeds(
1321             'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
1322         )
1323       '';
1324     };
1325
1326     smokeping = {
1327       exporterConfig = {
1328         enable = true;
1329         hosts = [ "127.0.0.1" ];
1330       };
1331       exporterTest = ''
1332         wait_for_unit("prometheus-smokeping-exporter.service")
1333         wait_for_open_port(9374)
1334         wait_until_succeeds(
1335             "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1336                 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source=""} '
1337             )
1338         )
1339         wait_until_succeeds(
1340             "curl -sSf localhost:9374/metrics | grep '{}'".format(
1341                 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source=""}'
1342             )
1343         )
1344       '';
1345     };
1346
1347     snmp = {
1348       exporterConfig = {
1349         enable = true;
1350         configuration.default = {
1351           version = 2;
1352           auth.community = "public";
1353         };
1354       };
1355       exporterTest = ''
1356         wait_for_unit("prometheus-snmp-exporter.service")
1357         wait_for_open_port(9116)
1358         succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1359       '';
1360     };
1361
1362     sql = {
1363       exporterConfig = {
1364         configuration.jobs.points = {
1365           interval = "1m";
1366           connections = [
1367             "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1368           ];
1369           queries = {
1370             points = {
1371               labels = [ "name" ];
1372               help = "Amount of points accumulated per person";
1373               values = [ "amount" ];
1374               query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1375             };
1376           };
1377         };
1378         enable = true;
1379         user = "prometheus-sql-exporter";
1380       };
1381       metricProvider = {
1382         services.postgresql = {
1383           enable = true;
1384           initialScript = builtins.toFile "init.sql" ''
1385             CREATE DATABASE data;
1386             \c data;
1387             CREATE TABLE points (amount INT, name TEXT);
1388             INSERT INTO points(amount, name) VALUES (1, 'jack');
1389             INSERT INTO points(amount, name) VALUES (2, 'jill');
1390             INSERT INTO points(amount, name) VALUES (3, 'jack');
1391
1392             CREATE USER "prometheus-sql-exporter";
1393             GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1394             GRANT SELECT ON points TO "prometheus-sql-exporter";
1395           '';
1396         };
1397         systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1398       };
1399       exporterTest = ''
1400         wait_for_unit("prometheus-sql-exporter.service")
1401         wait_for_open_port(9237)
1402         succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1403       '';
1404     };
1405
1406     statsd = {
1407       exporterConfig = {
1408         enable = true;
1409       };
1410       exporterTest = ''
1411         wait_for_unit("prometheus-statsd-exporter.service")
1412         wait_for_open_port(9102)
1413         succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
1414         wait_until_succeeds(
1415           "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
1416           curl http://localhost:9102/metrics | grep 'test_udp 1'",
1417           timeout=10
1418         )
1419         wait_until_succeeds(
1420           "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
1421           curl http://localhost:9102/metrics | grep 'test_tcp 1'",
1422           timeout=10
1423         )
1424       '';
1425     };
1426
1427     surfboard = {
1428       exporterConfig = {
1429         enable = true;
1430         modemAddress = "localhost";
1431       };
1432       metricProvider = {
1433         systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1434         services.nginx = {
1435           enable = true;
1436           virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1437             return 204;
1438           '';
1439         };
1440       };
1441       exporterTest = ''
1442         wait_for_unit("nginx.service")
1443         wait_for_open_port(80)
1444         wait_for_unit("prometheus-surfboard-exporter.service")
1445         wait_for_open_port(9239)
1446         succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1447       '';
1448     };
1449
1450     systemd = {
1451       exporterConfig = {
1452         enable = true;
1453
1454         extraFlags = [
1455           "--systemd.collector.enable-restart-count"
1456         ];
1457       };
1458       metricProvider = { };
1459       exporterTest = ''
1460         wait_for_unit("prometheus-systemd-exporter.service")
1461         wait_for_open_port(9558)
1462         wait_until_succeeds(
1463             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1464                 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1465             )
1466         )
1467         succeed(
1468             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1469                 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1470             )
1471         )
1472       '';
1473     };
1474
1475     tor = {
1476       exporterConfig = {
1477         enable = true;
1478       };
1479       metricProvider = {
1480         # Note: this does not connect the test environment to the Tor network.
1481         # Client, relay, bridge or exit connectivity are disabled by default.
1482         services.tor.enable = true;
1483         services.tor.settings.ControlPort = 9051;
1484       };
1485       exporterTest = ''
1486         wait_for_unit("tor.service")
1487         wait_for_open_port(9051)
1488         wait_for_unit("prometheus-tor-exporter.service")
1489         wait_for_open_port(9130)
1490         succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
1491       '';
1492     };
1493
1494     unpoller = {
1495       nodeName = "unpoller";
1496       exporterConfig.enable = true;
1497       exporterConfig.controllers = [{ }];
1498       exporterTest = ''
1499         wait_until_succeeds(
1500             'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
1501         )
1502       '';
1503     };
1504
1505     unbound = {
1506       exporterConfig = {
1507         enable = true;
1508         unbound.host = "unix:///run/unbound/unbound.ctl";
1509       };
1510       metricProvider = {
1511         services.unbound = {
1512           enable = true;
1513           localControlSocketPath = "/run/unbound/unbound.ctl";
1514         };
1515         systemd.services.prometheus-unbound-exporter.serviceConfig = {
1516           SupplementaryGroups = [ "unbound" ];
1517         };
1518       };
1519       exporterTest = ''
1520         wait_for_unit("unbound.service")
1521         wait_for_unit("prometheus-unbound-exporter.service")
1522         wait_for_open_port(9167)
1523         wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1524       '';
1525     };
1526
1527     v2ray = {
1528       exporterConfig = {
1529         enable = true;
1530       };
1531
1532       metricProvider = {
1533         systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1534         services.v2ray = {
1535           enable = true;
1536           config = {
1537             stats = {};
1538             api = {
1539               tag = "api";
1540               services = [ "StatsService" ];
1541             };
1542             inbounds = [
1543               {
1544                 port = 1080;
1545                 listen = "127.0.0.1";
1546                 protocol = "http";
1547               }
1548               {
1549                 listen = "127.0.0.1";
1550                 port = 54321;
1551                 protocol = "dokodemo-door";
1552                 settings = { address = "127.0.0.1"; };
1553                 tag = "api";
1554               }
1555             ];
1556             outbounds = [
1557               {
1558                 protocol = "freedom";
1559               }
1560               {
1561                 protocol = "freedom";
1562                 settings = {};
1563                 tag = "api";
1564               }
1565             ];
1566             routing = {
1567               strategy = "rules";
1568               settings = {
1569                 rules = [
1570                   {
1571                     inboundTag = [ "api" ];
1572                     outboundTag = "api";
1573                     type = "field";
1574                   }
1575                 ];
1576               };
1577             };
1578           };
1579         };
1580       };
1581       exporterTest = ''
1582         wait_for_unit("prometheus-v2ray-exporter.service")
1583         wait_for_open_port(9299)
1584         succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1585       '';
1586     };
1587
1588     varnish = {
1589       exporterConfig = {
1590         enable = true;
1591         instance = "/var/spool/varnish/varnish";
1592         group = "varnish";
1593       };
1594       metricProvider = {
1595         systemd.services.prometheus-varnish-exporter.after = [
1596           "varnish.service"
1597         ];
1598         services.varnish = {
1599           enable = true;
1600           config = ''
1601             vcl 4.0;
1602             backend default {
1603               .host = "127.0.0.1";
1604               .port = "80";
1605             }
1606           '';
1607         };
1608       };
1609       exporterTest = ''
1610         wait_for_unit("prometheus-varnish-exporter.service")
1611         wait_for_open_port(6081)
1612         wait_for_open_port(9131)
1613         succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1614       '';
1615     };
1616
1617     wireguard = let
1618       snakeoil = import ./wireguard/snakeoil-keys.nix;
1619       publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
1620     in
1621       {
1622         exporterConfig.enable = true;
1623         metricProvider = {
1624           networking.wireguard.interfaces.wg0 = {
1625             ips = [ "10.23.42.1/32" "fc00::1/128" ];
1626             listenPort = 23542;
1627
1628             inherit (snakeoil.peer0) privateKey;
1629
1630             peers = singleton {
1631               allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1632
1633               inherit (snakeoil.peer1) publicKey;
1634             };
1635           };
1636           systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1637         };
1638         exporterTest = ''
1639           wait_for_unit("prometheus-wireguard-exporter.service")
1640           wait_for_open_port(9586)
1641           wait_until_succeeds(
1642               "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
1643           )
1644         '';
1645       };
1646
1647     zfs = {
1648       exporterConfig = {
1649         enable = true;
1650       };
1651       metricProvider = {
1652         boot.supportedFilesystems = [ "zfs" ];
1653         networking.hostId = "7327ded7";
1654       };
1655       exporterTest = ''
1656         wait_for_unit("prometheus-zfs-exporter.service")
1657         wait_for_unit("zfs.target")
1658         wait_for_open_port(9134)
1659         wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1660       '';
1661     };
1662   };
1663 in
1664 mapAttrs
1665   (exporter: testConfig: (makeTest (
1666     let
1667       nodeName = testConfig.nodeName or exporter;
1668
1669     in
1670     {
1671       name = "prometheus-${exporter}-exporter";
1672
1673       nodes.${nodeName} = mkMerge [{
1674         services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1675       } testConfig.metricProvider or { }];
1676
1677       testScript = ''
1678         ${nodeName}.start()
1679         ${concatStringsSep "\n" (map (line:
1680           if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")")
1681           then line
1682           else "${nodeName}.${line}"
1683         ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1684         ${nodeName}.shutdown()
1685       '';
1686
1687       meta = with maintainers; {
1688         maintainers = [ willibutz ];
1689       };
1690     }
1691   )))
1692   exporterTests