nixos/tests/prometheus-exporters.nix

   1 { system ? builtins.currentSystem
   2 , config ? { }
   3 , pkgs ? import ../.. { inherit system config; }
   4 }:
   5
   6 let
   7   inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
   8   inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
   9     removeSuffix replaceStrings singleton splitString makeBinPath;
  10
  11   /*
  12     * The attrset `exporterTests` contains one attribute
  13     * for each exporter test. Each of these attributes
  14     * is expected to be an attrset containing:
  15     *
  16     *  `exporterConfig`:
  17     *    this attribute set contains config for the exporter itself
  18     *
  19     *  `exporterTest`
  20     *    this attribute set contains test instructions
  21     *
  22     *  `metricProvider` (optional)
  23     *    this attribute contains additional machine config
  24     *
  25     *  `nodeName` (optional)
  26     *    override an incompatible testnode name
  27     *
  28     *  Example:
  29     *    exporterTests.<exporterName> = {
  30     *      exporterConfig = {
  31     *        enable = true;
  32     *      };
  33     *      metricProvider = {
  34     *        services.<metricProvider>.enable = true;
  35     *      };
  36     *      exporterTest = ''
  37     *        wait_for_unit("prometheus-<exporterName>-exporter.service")
  38     *        wait_for_open_port(1234)
  39     *        succeed("curl -sSf 'localhost:1234/metrics'")
  40     *      '';
  41     *    };
  42     *
  43     *  # this would generate the following test config:
  44     *
  45     *    nodes.<exporterName> = {
  46     *      services.prometheus.<exporterName> = {
  47     *        enable = true;
  48     *      };
  49     *      services.<metricProvider>.enable = true;
  50     *    };
  51     *
  52     *    testScript = ''
  53     *      <exporterName>.start()
  54     *      <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
  55     *      <exporterName>.wait_for_open_port(1234)
  56     *      <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
  57     *      <exporterName>.shutdown()
  58     *    '';
  59   */
  60
  61   exporterTests = {
  62     apcupsd = {
  63       exporterConfig = {
  64         enable = true;
  65       };
  66       metricProvider = {
  67         services.apcupsd.enable = true;
  68       };
  69       exporterTest = ''
  70         wait_for_unit("apcupsd.service")
  71         wait_for_open_port(3551)
  72         wait_for_unit("prometheus-apcupsd-exporter.service")
  73         wait_for_open_port(9162)
  74         succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
  75       '';
  76     };
  77
  78     artifactory = {
  79       exporterConfig = {
  80         enable = true;
  81         artiUsername = "artifactory-username";
  82         artiPassword = "artifactory-password";
  83       };
  84       exporterTest = ''
  85         wait_for_unit("prometheus-artifactory-exporter.service")
  86         wait_for_open_port(9531)
  87         succeed(
  88             "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
  89         )
  90       '';
  91     };
  92
  93     bind = {
  94       exporterConfig = {
  95         enable = true;
  96       };
  97       metricProvider = {
  98         services.bind.enable = true;
  99         services.bind.extraConfig = ''
 100           statistics-channels {
 101             inet 127.0.0.1 port 8053 allow { localhost; };
 102           };
 103         '';
 104       };
 105       exporterTest = ''
 106         wait_for_unit("prometheus-bind-exporter.service")
 107         wait_for_open_port(9119)
 108         succeed(
 109             "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
 110         )
 111       '';
 112     };
 113
 114     bird = {
 115       exporterConfig = {
 116         enable = true;
 117       };
 118       metricProvider = {
 119         services.bird2.enable = true;
 120         services.bird2.config = ''
 121           router id 127.0.0.1;
 122
 123           protocol kernel MyObviousTestString {
 124             ipv4 {
 125               import all;
 126               export none;
 127             };
 128           }
 129
 130           protocol device {
 131           }
 132         '';
 133       };
 134       exporterTest = ''
 135         wait_for_unit("prometheus-bird-exporter.service")
 136         wait_for_open_port(9324)
 137         wait_until_succeeds(
 138             "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
 139         )
 140       '';
 141     };
 142
 143     bitcoin = {
 144       exporterConfig = {
 145         enable = true;
 146         rpcUser = "bitcoinrpc";
 147         rpcPasswordFile = pkgs.writeText "password" "hunter2";
 148       };
 149       metricProvider = {
 150         services.bitcoind.default.enable = true;
 151         services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
 152       };
 153       exporterTest = ''
 154         wait_for_unit("prometheus-bitcoin-exporter.service")
 155         wait_for_unit("bitcoind-default.service")
 156         wait_for_open_port(9332)
 157         succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
 158       '';
 159     };
 160
 161     blackbox = {
 162       exporterConfig = {
 163         enable = true;
 164         configFile = pkgs.writeText "config.yml" (builtins.toJSON {
 165           modules.icmp_v6 = {
 166             prober = "icmp";
 167             icmp.preferred_ip_protocol = "ip6";
 168           };
 169         });
 170       };
 171       exporterTest = ''
 172         wait_for_unit("prometheus-blackbox-exporter.service")
 173         wait_for_open_port(9115)
 174         succeed(
 175             "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
 176         )
 177       '';
 178     };
 179
 180     borgmatic = {
 181       exporterConfig = {
 182         enable = true;
 183         user = "root";
 184       };
 185       metricProvider = {
 186         services.borgmatic.enable = true;
 187         services.borgmatic.settings.source_directories = [ "/home" ];
 188         services.borgmatic.settings.repositories = [ { label = "local"; path = "/var/backup"; } ];
 189         services.borgmatic.settings.keep_daily = 10;
 190       };
 191       exporterTest = ''
 192         succeed("borgmatic rcreate -e none")
 193         succeed("borgmatic")
 194         wait_for_unit("prometheus-borgmatic-exporter.service")
 195         wait_for_open_port(9996)
 196         succeed("curl -sSf localhost:9996/metrics | grep 'borg_total_backups{repository=\"/var/backup\"} 1'")
 197       '';
 198     };
 199
 200     collectd = {
 201       exporterConfig = {
 202         enable = true;
 203         extraFlags = [ "--web.collectd-push-path /collectd" ];
 204       };
 205       exporterTest = let postData = replaceStrings [ "\n" ] [ "" ] ''
 206         [{
 207           "values":[23],
 208           "dstypes":["gauge"],
 209           "type":"gauge",
 210           "interval":1000,
 211           "host":"testhost",
 212           "plugin":"testplugin",
 213           "time":DATE
 214         }]
 215       ''; in
 216         ''
 217           wait_for_unit("prometheus-collectd-exporter.service")
 218           wait_for_open_port(9103)
 219           succeed(
 220               'echo \'${postData}\'> /tmp/data.json'
 221           )
 222           succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
 223           succeed(
 224               "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
 225           )
 226           succeed(
 227               "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
 228           )
 229         '';
 230     };
 231
 232     deluge = {
 233       exporterConfig = {
 234         enable = true;
 235         port = 1234;
 236         listenAddress = "127.0.0.1";
 237
 238         delugeUser = "user";
 239         delugePort = 2345;
 240         delugePasswordFile = pkgs.writeText "password" "weak_password";
 241       };
 242       metricProvider = {
 243         services.deluge.enable = true;
 244         services.deluge.declarative = true;
 245         services.deluge.config.daemon_port = 2345;
 246         services.deluge.authFile = pkgs.writeText "authFile" ''
 247         localclient:abcdef:10
 248         user:weak_password:10
 249         '';
 250       };
 251       exporterTest = ''
 252         wait_for_unit("deluged.service")
 253         wait_for_open_port(2345)
 254         wait_for_unit("prometheus-deluge-exporter.service")
 255         wait_for_open_port(1234)
 256         succeed("curl -sSf http://localhost:1234 | grep 'deluge_torrents'")
 257       '';
 258     };
 259
 260     dnsmasq = {
 261       exporterConfig = {
 262         enable = true;
 263         leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
 264       };
 265       metricProvider = {
 266         services.dnsmasq.enable = true;
 267       };
 268       exporterTest = ''
 269         wait_for_unit("dnsmasq.service")
 270         wait_for_open_port(53)
 271         wait_for_file("/var/lib/dnsmasq/dnsmasq.leases")
 272         wait_for_unit("prometheus-dnsmasq-exporter.service")
 273         wait_for_open_port(9153)
 274         succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
 275       '';
 276     };
 277
 278     dnssec = {
 279       exporterConfig = {
 280         enable = true;
 281         configuration = {
 282           records = [
 283             {
 284               zone = "example.com";
 285               record = "@";
 286               type = "SOA";
 287             }
 288           ];
 289         };
 290         resolvers = [ "127.0.0.1:53" ];
 291       };
 292       metricProvider = {
 293         services.knot = {
 294           enable = true;
 295           settingsFile = pkgs.writeText "knot.conf" ''
 296             server:
 297               listen: 127.0.0.1@53
 298             template:
 299               - id: default
 300                 storage: ${pkgs.buildEnv {
 301                   name = "zones";
 302                   paths = [(pkgs.writeTextDir "example.com.zone" ''
 303                     @ SOA ns1.example.com. noc.example.com. 2024032401 86400 7200 3600000 172800
 304                     @       NS      ns1
 305                     ns1     A       192.168.0.1
 306                   '')];
 307                 }}
 308                 zonefile-load: difference
 309                 zonefile-sync: -1
 310             zone:
 311               - domain: example.com
 312                 file: example.com.zone
 313                 dnssec-signing: on
 314           '';
 315         };
 316       };
 317       exporterTest = ''
 318         wait_for_unit("knot.service")
 319         wait_for_open_port(53)
 320         wait_for_unit("prometheus-dnssec-exporter.service")
 321         wait_for_open_port(9204)
 322         succeed("curl -sSf http://localhost:9204/metrics | grep 'example.com'")
 323       '';
 324     };
 325
 326     # Access to WHOIS server is required to properly test this exporter, so
 327     # just perform basic sanity check that the exporter is running and returns
 328     # a failure.
 329     domain = {
 330       exporterConfig = {
 331         enable = true;
 332       };
 333       exporterTest = ''
 334         wait_for_unit("prometheus-domain-exporter.service")
 335         wait_for_open_port(9222)
 336         succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
 337       '';
 338     };
 339
 340     dovecot = {
 341       exporterConfig = {
 342         enable = true;
 343         scopes = [ "global" ];
 344         socketPath = "/var/run/dovecot2/old-stats";
 345         user = "root"; # <- don't use user root in production
 346       };
 347       metricProvider = {
 348         services.dovecot2.enable = true;
 349       };
 350       exporterTest = ''
 351         wait_for_unit("prometheus-dovecot-exporter.service")
 352         wait_for_open_port(9166)
 353         succeed(
 354             "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
 355         )
 356       '';
 357     };
 358
 359     fastly = {
 360       exporterConfig = {
 361         enable = true;
 362         tokenPath = pkgs.writeText "token" "abc123";
 363       };
 364
 365       exporterTest = ''
 366         wait_for_unit("prometheus-fastly-exporter.service")
 367         wait_for_open_port(9118)
 368       '';
 369     };
 370
 371     fritzbox = {
 372       # TODO add proper test case
 373       exporterConfig = {
 374         enable = true;
 375       };
 376       exporterTest = ''
 377         wait_for_unit("prometheus-fritzbox-exporter.service")
 378         wait_for_open_port(9133)
 379         succeed(
 380             "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
 381         )
 382       '';
 383     };
 384
 385     graphite = {
 386       exporterConfig = {
 387         enable = true;
 388         port = 9108;
 389         graphitePort = 9109;
 390         mappingSettings.mappings = [{
 391           match = "test.*.*";
 392           name = "testing";
 393           labels = {
 394             protocol = "$1";
 395             author = "$2";
 396           };
 397         }];
 398       };
 399       exporterTest = ''
 400         wait_for_unit("prometheus-graphite-exporter.service")
 401         wait_for_open_port(9108)
 402         wait_for_open_port(9109)
 403         succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
 404         succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
 405       '';
 406     };
 407
 408     idrac = {
 409       exporterConfig = {
 410         enable = true;
 411         port = 9348;
 412         configuration = {
 413           hosts = {
 414             default = { username = "username"; password = "password"; };
 415           };
 416         };
 417       };
 418       exporterTest = ''
 419         wait_for_unit("prometheus-idrac-exporter.service")
 420         wait_for_open_port(9348)
 421         wait_until_succeeds("curl localhost:9348")
 422       '';
 423     };
 424
 425     influxdb = {
 426       exporterConfig = {
 427         enable = true;
 428         sampleExpiry = "3s";
 429       };
 430       exporterTest = ''
 431         wait_for_unit("prometheus-influxdb-exporter.service")
 432         wait_for_open_port(9122)
 433         succeed(
 434           "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
 435         )
 436         succeed(
 437           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 438         )
 439         execute("sleep 5")
 440         fail(
 441           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 442         )
 443       '';
 444     };
 445
 446     ipmi = {
 447       exporterConfig = {
 448         enable = true;
 449       };
 450       exporterTest = ''
 451         wait_for_unit("prometheus-ipmi-exporter.service")
 452         wait_for_open_port(9290)
 453         succeed(
 454           "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
 455         )
 456       '';
 457     };
 458
 459     jitsi = {
 460       exporterConfig = {
 461         enable = true;
 462       };
 463       metricProvider = {
 464         systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
 465         services.jitsi-videobridge = {
 466           enable = true;
 467           colibriRestApi = true;
 468         };
 469       };
 470       exporterTest = ''
 471         wait_for_unit("jitsi-videobridge2.service")
 472         wait_for_open_port(8080)
 473         wait_for_unit("prometheus-jitsi-exporter.service")
 474         wait_for_open_port(9700)
 475         wait_until_succeeds(
 476             'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
 477         )
 478         succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
 479       '';
 480     };
 481
 482     json = {
 483       exporterConfig = {
 484         enable = true;
 485         configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
 486           modules = {
 487             default = {
 488               metrics = [
 489                 { name = "json_test_metric"; path = "{ .test }"; }
 490               ];
 491             };
 492           };
 493         });
 494       };
 495       metricProvider = {
 496         systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
 497         services.nginx = {
 498           enable = true;
 499           virtualHosts.localhost.locations."/".extraConfig = ''
 500             return 200 "{\"test\":1}";
 501           '';
 502         };
 503       };
 504       exporterTest = ''
 505         wait_for_unit("nginx.service")
 506         wait_for_open_port(80)
 507         wait_for_unit("prometheus-json-exporter.service")
 508         wait_for_open_port(7979)
 509         succeed(
 510             "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
 511         )
 512       '';
 513     };
 514
 515     knot = {
 516       exporterConfig = {
 517         enable = true;
 518       };
 519       metricProvider = {
 520         services.knot = {
 521           enable = true;
 522           extraArgs = [ "-v" ];
 523           settingsFile = pkgs.writeText "knot.conf" ''
 524             server:
 525               listen: 127.0.0.1@53
 526
 527             template:
 528               - id: default
 529                 global-module: mod-stats
 530                 dnssec-signing: off
 531                 zonefile-sync: -1
 532                 zonefile-load: difference
 533                 storage: ${pkgs.buildEnv {
 534                   name = "foo";
 535                   paths = [
 536                     (pkgs.writeTextDir "test.zone" ''
 537                       @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
 538                       @       NS      ns1
 539                       @       NS      ns2
 540                       ns1     A       192.168.0.1
 541                     '')
 542                   ];
 543                 }}
 544
 545             mod-stats:
 546               - id: custom
 547                 edns-presence: on
 548                 query-type: on
 549
 550             zone:
 551               - domain: test
 552                 file: test.zone
 553                 module: mod-stats/custom
 554           '';
 555         };
 556       };
 557       exporterTest = ''
 558         wait_for_unit("knot.service")
 559         wait_for_unit("prometheus-knot-exporter.service")
 560         wait_for_open_port(9433)
 561         succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
 562       '';
 563     };
 564
 565     keylight = {
 566       # A hardware device is required to properly test this exporter, so just
 567       # perform a couple of basic sanity checks that the exporter is running
 568       # and requires a target, but cannot reach a specified target.
 569       exporterConfig = {
 570         enable = true;
 571       };
 572       exporterTest = ''
 573         wait_for_unit("prometheus-keylight-exporter.service")
 574         wait_for_open_port(9288)
 575         succeed(
 576             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
 577         )
 578         succeed(
 579             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
 580         )
 581       '';
 582     };
 583
 584     lnd = {
 585       exporterConfig = {
 586         enable = true;
 587         lndTlsPath = "/var/lib/lnd/tls.cert";
 588         lndMacaroonDir = "/var/lib/lnd";
 589         extraFlags = [ "--lnd.network=regtest" ];
 590       };
 591       metricProvider = {
 592         systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
 593         systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
 594         services.bitcoind.regtest = {
 595           enable = true;
 596           extraConfig = ''
 597             rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
 598             zmqpubrawblock=tcp://127.0.0.1:28332
 599             zmqpubrawtx=tcp://127.0.0.1:28333
 600           '';
 601           extraCmdlineOptions = [ "-regtest" ];
 602         };
 603         systemd.services.lnd = {
 604           serviceConfig.ExecStart = ''
 605             ${pkgs.lnd}/bin/lnd \
 606               --datadir=/var/lib/lnd \
 607               --tlscertpath=/var/lib/lnd/tls.cert \
 608               --tlskeypath=/var/lib/lnd/tls.key \
 609               --logdir=/var/log/lnd \
 610               --bitcoin.active \
 611               --bitcoin.regtest \
 612               --bitcoin.node=bitcoind \
 613               --bitcoind.rpcuser=bitcoinrpc \
 614               --bitcoind.rpcpass=hunter2 \
 615               --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
 616               --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
 617               --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
 618           '';
 619           serviceConfig.StateDirectory = "lnd";
 620           wantedBy = [ "multi-user.target" ];
 621           after = [ "network.target" ];
 622         };
 623         # initialize wallet, creates macaroon needed by exporter
 624         systemd.services.lnd.postStart = ''
 625           ${pkgs.curl}/bin/curl \
 626             --retry 20 \
 627             --retry-delay 1 \
 628             --retry-connrefused \
 629             --cacert /var/lib/lnd/tls.cert \
 630             -X GET \
 631             https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
 632           ${pkgs.curl}/bin/curl \
 633             --retry 20 \
 634             --retry-delay 1 \
 635             --retry-connrefused \
 636             --cacert /var/lib/lnd/tls.cert \
 637             -X POST \
 638             -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
 639             https://localhost:8080/v1/initwallet
 640         '';
 641       };
 642       exporterTest = ''
 643         wait_for_unit("lnd.service")
 644         wait_for_open_port(10009)
 645         wait_for_unit("prometheus-lnd-exporter.service")
 646         wait_for_open_port(9092)
 647         succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
 648       '';
 649     };
 650
 651     mail = {
 652       exporterConfig = {
 653         enable = true;
 654         configuration = {
 655           monitoringInterval = "2s";
 656           mailCheckTimeout = "10s";
 657           servers = [{
 658             name = "testserver";
 659             server = "localhost";
 660             port = 25;
 661             from = "mail-exporter@localhost";
 662             to = "mail-exporter@localhost";
 663             detectionDir = "/var/spool/mail/mail-exporter/new";
 664           }];
 665         };
 666       };
 667       metricProvider = {
 668         services.postfix.enable = true;
 669         systemd.services.prometheus-mail-exporter = {
 670           after = [ "postfix.service" ];
 671           requires = [ "postfix.service" ];
 672           serviceConfig = {
 673             ExecStartPre = [
 674               "${pkgs.writeShellScript "create-maildir" ''
 675                 mkdir -p -m 0700 mail-exporter/new
 676               ''}"
 677             ];
 678             ProtectHome = true;
 679             ReadOnlyPaths = "/";
 680             ReadWritePaths = "/var/spool/mail";
 681             WorkingDirectory = "/var/spool/mail";
 682           };
 683         };
 684         users.users.mailexporter = {
 685           isSystemUser = true;
 686           group = "mailexporter";
 687         };
 688         users.groups.mailexporter = {};
 689       };
 690       exporterTest = ''
 691         wait_for_unit("postfix.service")
 692         wait_for_unit("prometheus-mail-exporter.service")
 693         wait_for_open_port(9225)
 694         wait_until_succeeds(
 695             "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
 696         )
 697       '';
 698     };
 699
 700     mikrotik = {
 701       exporterConfig = {
 702         enable = true;
 703         extraFlags = [ "-timeout=1s" ];
 704         configuration = {
 705           devices = [
 706             {
 707               name = "router";
 708               address = "192.168.42.48";
 709               user = "prometheus";
 710               password = "shh";
 711             }
 712           ];
 713           features = {
 714             bgp = true;
 715             dhcp = true;
 716             dhcpl = true;
 717             dhcpv6 = true;
 718             health = true;
 719             routes = true;
 720             poe = true;
 721             pools = true;
 722             optics = true;
 723             w60g = true;
 724             wlansta = true;
 725             wlanif = true;
 726             monitor = true;
 727             ipsec = true;
 728           };
 729         };
 730       };
 731       exporterTest = ''
 732         wait_for_unit("prometheus-mikrotik-exporter.service")
 733         wait_for_open_port(9436)
 734         succeed(
 735             "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
 736         )
 737       '';
 738     };
 739
 740     modemmanager = {
 741       exporterConfig = {
 742         enable = true;
 743         refreshRate = "10s";
 744       };
 745       metricProvider = {
 746         # ModemManager is installed when NetworkManager is enabled. Ensure it is
 747         # started and is wanted by NM and the exporter to start everything up
 748         # in the right order.
 749         networking.networkmanager.enable = true;
 750         systemd.services.ModemManager = {
 751           enable = true;
 752           wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
 753         };
 754       };
 755       exporterTest = ''
 756         wait_for_unit("ModemManager.service")
 757         wait_for_unit("prometheus-modemmanager-exporter.service")
 758         wait_for_open_port(9539)
 759         succeed(
 760             "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
 761         )
 762       '';
 763     };
 764
 765     mysqld = {
 766       exporterConfig = {
 767         enable = true;
 768         runAsLocalSuperUser = true;
 769         configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
 770           [client]
 771           user = exporter
 772           password = snakeoilpassword
 773         '';
 774       };
 775       metricProvider = {
 776         services.mysql = {
 777           enable = true;
 778           package = pkgs.mariadb;
 779           initialScript = pkgs.writeText "mysql-init-script.sql" ''
 780             CREATE USER 'exporter'@'localhost'
 781             IDENTIFIED BY 'snakeoilpassword'
 782             WITH MAX_USER_CONNECTIONS 3;
 783             GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
 784           '';
 785         };
 786       };
 787       exporterTest = ''
 788         wait_for_unit("prometheus-mysqld-exporter.service")
 789         wait_for_open_port(9104)
 790         wait_for_unit("mysql.service")
 791         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 792         systemctl("stop mysql.service")
 793         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
 794         systemctl("start mysql.service")
 795         wait_for_unit("mysql.service")
 796         succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
 797       '';
 798     };
 799
 800     nextcloud = {
 801       exporterConfig = {
 802         enable = true;
 803         passwordFile = "/var/nextcloud-pwfile";
 804         url = "http://localhost";
 805       };
 806       metricProvider = {
 807         systemd.services.nc-pwfile =
 808           let
 809             passfile = (pkgs.writeText "pwfile" "snakeoilpw");
 810           in
 811           {
 812             requiredBy = [ "prometheus-nextcloud-exporter.service" ];
 813             before = [ "prometheus-nextcloud-exporter.service" ];
 814             serviceConfig.ExecStart = ''
 815               ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
 816             '';
 817           };
 818         services.nginx = {
 819           enable = true;
 820           virtualHosts."localhost" = {
 821             basicAuth.nextcloud-exporter = "snakeoilpw";
 822             locations."/" = {
 823               root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
 824               tryFiles = "/negative-space.json =404";
 825             };
 826           };
 827         };
 828       };
 829       exporterTest = ''
 830         wait_for_unit("nginx.service")
 831         wait_for_unit("prometheus-nextcloud-exporter.service")
 832         wait_for_open_port(9205)
 833         succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
 834       '';
 835     };
 836
 837     nginx = {
 838       exporterConfig = {
 839         enable = true;
 840         constLabels = [ "foo=bar" ];
 841       };
 842       metricProvider = {
 843         services.nginx = {
 844           enable = true;
 845           statusPage = true;
 846           virtualHosts."test".extraConfig = "return 204;";
 847         };
 848       };
 849       exporterTest = ''
 850         wait_for_unit("nginx.service")
 851         wait_for_unit("prometheus-nginx-exporter.service")
 852         wait_for_open_port(9113)
 853         succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up{foo=\"bar\"} 1'")
 854       '';
 855     };
 856
 857     nginxlog = {
 858       exporterConfig = {
 859         enable = true;
 860         group = "nginx";
 861         settings = {
 862           namespaces = [
 863             {
 864               name = "filelogger";
 865               source = {
 866                 files = [ "/var/log/nginx/filelogger.access.log" ];
 867               };
 868             }
 869             {
 870               name = "syslogger";
 871               source = {
 872                 syslog = {
 873                   listen_address = "udp://127.0.0.1:10000";
 874                   format = "rfc3164";
 875                   tags = [ "nginx" ];
 876                 };
 877               };
 878             }
 879           ];
 880         };
 881       };
 882       metricProvider = {
 883         services.nginx = {
 884           enable = true;
 885           httpConfig = ''
 886             server {
 887               listen 80;
 888               server_name filelogger.local;
 889               access_log /var/log/nginx/filelogger.access.log;
 890             }
 891             server {
 892               listen 81;
 893               server_name syslogger.local;
 894               access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
 895             }
 896           '';
 897         };
 898       };
 899       exporterTest = ''
 900         wait_for_unit("nginx.service")
 901         wait_for_unit("prometheus-nginxlog-exporter.service")
 902         wait_for_open_port(9117)
 903         wait_for_open_port(80)
 904         wait_for_open_port(81)
 905         succeed("curl http://localhost")
 906         execute("sleep 1")
 907         succeed(
 908             "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
 909         )
 910         succeed("curl http://localhost:81")
 911         execute("sleep 1")
 912         succeed(
 913             "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
 914         )
 915       '';
 916     };
 917
 918     node = {
 919       exporterConfig = {
 920         enable = true;
 921       };
 922       exporterTest = ''
 923         wait_for_unit("prometheus-node-exporter.service")
 924         wait_for_open_port(9100)
 925         succeed(
 926             "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
 927         )
 928       '';
 929     };
 930
 931     pgbouncer = {
 932       exporterConfig = {
 933         enable = true;
 934         connectionEnvFile = "${pkgs.writeText "connstr-env" ''
 935           PGBOUNCER_EXPORTER_CONNECTION_STRING=postgres://admin@localhost:6432/pgbouncer?sslmode=disable
 936         ''}";
 937       };
 938
 939       metricProvider = {
 940         services.postgresql.enable = true;
 941         services.pgbouncer = {
 942           enable = true;
 943           settings = {
 944             pgbouncer = {
 945               listen_addr = "*";
 946               auth_type = "any";
 947               max_client_conn = 99;
 948               # https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
 949               ignore_startup_parameters = "extra_float_digits";
 950             };
 951             databases = {
 952               postgres = concatStringsSep " " [
 953                 "host=/run/postgresql"
 954                 "port=5432"
 955                 "auth_user=postgres"
 956                 "dbname=postgres"
 957               ];
 958             };
 959           };
 960         };
 961       };
 962       exporterTest = ''
 963         wait_for_unit("postgresql.service")
 964         wait_for_unit("pgbouncer.service")
 965         wait_for_unit("prometheus-pgbouncer-exporter.service")
 966         wait_for_open_port(9127)
 967         succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
 968         succeed(
 969             "curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
 970         )
 971       '';
 972     };
 973
 974     php-fpm = {
 975       nodeName = "php_fpm";
 976       exporterConfig = {
 977         enable = true;
 978         environmentFile = pkgs.writeTextFile {
 979           name = "/tmp/prometheus-php-fpm-exporter.env";
 980           text = ''
 981             PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
 982           '';
 983         };
 984       };
 985       metricProvider = {
 986         users.users."php-fpm-exporter" = {
 987           isSystemUser = true;
 988           group  = "php-fpm-exporter";
 989         };
 990         users.groups."php-fpm-exporter" = {};
 991         services.phpfpm.pools."php-fpm-exporter" = {
 992           user = "php-fpm-exporter";
 993           group = "php-fpm-exporter";
 994           settings = {
 995             "pm" = "dynamic";
 996             "pm.max_children" = 32;
 997             "pm.max_requests" = 500;
 998             "pm.start_servers" = 2;
 999             "pm.min_spare_servers" = 2;
1000             "pm.max_spare_servers" = 5;
1001             "pm.status_path" = "/status";
1002             "listen" = "127.0.0.1:9000";
1003             "listen.allowed_clients" = "127.0.0.1";
1004           };
1005           phpEnv."PATH" = makeBinPath [ pkgs.php ];
1006         };
1007       };
1008       exporterTest = ''
1009         wait_for_unit("phpfpm-php-fpm-exporter.service")
1010         wait_for_unit("prometheus-php-fpm-exporter.service")
1011         succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
1012       '';
1013     };
1014
1015     ping = {
1016       exporterConfig = {
1017         enable = true;
1018
1019         settings = {
1020           targets = [ {
1021             "localhost" = {
1022               alias = "local machine";
1023               env = "prod";
1024               type = "domain";
1025             };
1026           } {
1027             "127.0.0.1" = {
1028               alias = "local machine";
1029               type = "v4";
1030             };
1031           } {
1032             "::1" = {
1033               alias = "local machine";
1034               type = "v6";
1035             };
1036           } {
1037             "google.com" = {};
1038           } ];
1039           dns = {};
1040           ping = {
1041             interval = "2s";
1042             timeout = "3s";
1043             history-size = 42;
1044             payload-size = 56;
1045           };
1046           log = {
1047             level = "warn";
1048           };
1049         };
1050       };
1051
1052       exporterTest = ''
1053         wait_for_unit("prometheus-ping-exporter.service")
1054         wait_for_open_port(9427)
1055         succeed("curl -sSf http://localhost:9427/metrics | grep 'ping_up{.*} 1'")
1056       '';
1057     };
1058
1059     postfix = {
1060       exporterConfig = {
1061         enable = true;
1062       };
1063       metricProvider = {
1064         services.postfix.enable = true;
1065       };
1066       exporterTest = ''
1067         wait_for_unit("prometheus-postfix-exporter.service")
1068         wait_for_file("/var/lib/postfix/queue/public/showq")
1069         wait_for_open_port(9154)
1070         wait_until_succeeds(
1071             "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
1072         )
1073         succeed(
1074             "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
1075         )
1076         succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
1077       '';
1078     };
1079
1080     postgres = {
1081       exporterConfig = {
1082         enable = true;
1083         runAsLocalSuperUser = true;
1084       };
1085       metricProvider = {
1086         services.postgresql.enable = true;
1087       };
1088       exporterTest = ''
1089         wait_for_unit("prometheus-postgres-exporter.service")
1090         wait_for_open_port(9187)
1091         wait_for_unit("postgresql.service")
1092         succeed(
1093             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1094         )
1095         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1096         systemctl("stop postgresql.service")
1097         succeed(
1098             "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
1099         )
1100         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
1101         systemctl("start postgresql.service")
1102         wait_for_unit("postgresql.service")
1103         succeed(
1104             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
1105         )
1106         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
1107       '';
1108     };
1109
1110     process = {
1111       exporterConfig = {
1112         enable = true;
1113         settings.process_names = [
1114           # Remove nix store path from process name
1115           { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
1116         ];
1117       };
1118       exporterTest = ''
1119         wait_for_unit("prometheus-process-exporter.service")
1120         wait_for_open_port(9256)
1121         wait_until_succeeds(
1122             "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
1123                 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
1124             )
1125         )
1126       '';
1127     };
1128
1129     pve = let
1130       pveExporterEnvFile = pkgs.writeTextFile {
1131         name = "pve.env";
1132         text = ''
1133           PVE_USER="test_user@pam"
1134           PVE_PASSWORD="hunter3"
1135           PVE_VERIFY_SSL="false"
1136         '';
1137       };
1138     in {
1139       exporterConfig = {
1140         enable = true;
1141         environmentFile = pveExporterEnvFile;
1142       };
1143       exporterTest = ''
1144         wait_for_unit("prometheus-pve-exporter.service")
1145         wait_for_open_port(9221)
1146         wait_until_succeeds("curl localhost:9221")
1147       '';
1148     };
1149
1150     py-air-control = {
1151       nodeName = "py_air_control";
1152       exporterConfig = {
1153         enable = true;
1154         deviceHostname = "127.0.0.1";
1155       };
1156       exporterTest = ''
1157         wait_for_unit("prometheus-py-air-control-exporter.service")
1158         wait_for_open_port(9896)
1159         succeed(
1160             "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
1161         )
1162       '';
1163     };
1164
1165     redis = {
1166       exporterConfig = {
1167         enable = true;
1168       };
1169       metricProvider.services.redis.servers."".enable = true;
1170       exporterTest = ''
1171         wait_for_unit("redis.service")
1172         wait_for_unit("prometheus-redis-exporter.service")
1173         wait_for_open_port(6379)
1174         wait_for_open_port(9121)
1175         wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1176       '';
1177     };
1178
1179     restic =
1180       let
1181         repository = "rest:http://127.0.0.1:8000";
1182         passwordFile = pkgs.writeText "restic-test-password" "test-password";
1183       in
1184       {
1185         exporterConfig = {
1186           enable = true;
1187           inherit repository passwordFile;
1188         };
1189         metricProvider = {
1190           services.restic.server = {
1191             enable = true;
1192             extraFlags = [ "--no-auth" ];
1193           };
1194           environment.systemPackages = [ pkgs.restic ];
1195         };
1196         exporterTest = ''
1197           # prometheus-restic-exporter.service fails without initialised repository
1198           systemctl("stop prometheus-restic-exporter.service")
1199
1200           # Initialise the repository
1201           wait_for_unit("restic-rest-server.service")
1202           wait_for_open_port(8000)
1203           succeed("restic init --repo ${repository} --password-file ${passwordFile}")
1204
1205           systemctl("start prometheus-restic-exporter.service")
1206           wait_for_unit("prometheus-restic-exporter.service")
1207           wait_for_open_port(9753)
1208           wait_until_succeeds("curl -sSf localhost:9753/metrics | grep 'restic_check_success 1.0'")
1209         '';
1210       };
1211
1212     rspamd = {
1213       exporterConfig = {
1214         enable = true;
1215       };
1216       metricProvider = {
1217         services.rspamd.enable = true;
1218       };
1219       exporterTest = ''
1220         wait_for_unit("rspamd.service")
1221         wait_for_unit("prometheus-rspamd-exporter.service")
1222         wait_for_open_port(11334)
1223         wait_for_open_port(7980)
1224         wait_until_succeeds(
1225             "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1226         )
1227       '';
1228     };
1229
1230     rtl_433 = {
1231       exporterConfig = {
1232         enable = true;
1233       };
1234       metricProvider = {
1235         # Mock rtl_433 binary to return a dummy metric stream.
1236         nixpkgs.overlays = [
1237           (self: super: {
1238             rtl_433 = self.runCommand "rtl_433" { } ''
1239               mkdir -p "$out/bin"
1240               cat <<EOF > "$out/bin/rtl_433"
1241               #!/bin/sh
1242               while true; do
1243                 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1244                 sleep 4
1245               done
1246               EOF
1247               chmod +x "$out/bin/rtl_433"
1248             '';
1249           })
1250         ];
1251       };
1252       exporterTest = ''
1253         wait_for_unit("prometheus-rtl_433-exporter.service")
1254         wait_for_open_port(9550)
1255         wait_until_succeeds(
1256             "curl -sSf localhost:9550/metrics | grep '{}'".format(
1257                 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1258             )
1259         )
1260       '';
1261     };
1262
1263     sabnzbd = {
1264       exporterConfig = {
1265         enable = true;
1266         servers = [{
1267           baseUrl = "http://localhost:8080";
1268           apiKeyFile = "/var/sabnzbd-apikey";
1269         }];
1270       };
1271
1272       metricProvider = {
1273         services.sabnzbd.enable = true;
1274
1275         # unrar is required for sabnzbd
1276         nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (pkgs.lib.getName pkg) [ "unrar" ];
1277
1278         # extract the generated api key before starting
1279         systemd.services.sabnzbd-apikey = {
1280           requires = [ "sabnzbd.service" ];
1281           after = [ "sabnzbd.service" ];
1282           requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
1283           before = [ "prometheus-sabnzbd-exporter.service" ];
1284           script = ''
1285             grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
1286           '';
1287         };
1288       };
1289
1290       exporterTest = ''
1291         wait_for_unit("sabnzbd.service")
1292         wait_for_unit("prometheus-sabnzbd-exporter.service")
1293         wait_for_open_port(8080)
1294         wait_for_open_port(9387)
1295         wait_until_succeeds(
1296             "curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
1297         )
1298       '';
1299     };
1300
1301     scaphandre = {
1302       exporterConfig = {
1303         enable = true;
1304       };
1305       metricProvider = {
1306         boot.kernelModules = [ "intel_rapl_common" ];
1307       };
1308       exporterTest = ''
1309         wait_for_unit("prometheus-scaphandre-exporter.service")
1310         wait_for_open_port(8080)
1311         wait_until_succeeds(
1312             "curl -sSf 'localhost:8080/metrics'"
1313         )
1314       '';
1315     };
1316
1317     shelly = {
1318       exporterConfig = {
1319         enable = true;
1320         metrics-file = "${pkgs.writeText "test.json" ''{}''}";
1321       };
1322       exporterTest = ''
1323         wait_for_unit("prometheus-shelly-exporter.service")
1324         wait_for_open_port(9784)
1325         wait_until_succeeds(
1326             "curl -sSf 'localhost:9784/metrics'"
1327         )
1328       '';
1329     };
1330
1331     script = {
1332       exporterConfig = {
1333         enable = true;
1334         settings.scripts = [
1335           { name = "success"; script = "sleep 1"; }
1336         ];
1337       };
1338       exporterTest = ''
1339         wait_for_unit("prometheus-script-exporter.service")
1340         wait_for_open_port(9172)
1341         wait_until_succeeds(
1342             "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1343                 'script_success{script="success"} 1'
1344             )
1345         )
1346       '';
1347     };
1348
1349     smartctl = {
1350       exporterConfig = {
1351         enable = true;
1352         devices = [
1353           "/dev/vda"
1354         ];
1355       };
1356       exporterTest = ''
1357         wait_until_succeeds(
1358             'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
1359         )
1360       '';
1361     };
1362
1363     smokeping = {
1364       exporterConfig = {
1365         enable = true;
1366         hosts = [ "127.0.0.1" ];
1367       };
1368       exporterTest = ''
1369         wait_for_unit("prometheus-smokeping-exporter.service")
1370         wait_for_open_port(9374)
1371         wait_until_succeeds(
1372             "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1373                 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source=""} '
1374             )
1375         )
1376         wait_until_succeeds(
1377             "curl -sSf localhost:9374/metrics | grep '{}'".format(
1378                 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source=""}'
1379             )
1380         )
1381       '';
1382     };
1383
1384     snmp = {
1385       exporterConfig = {
1386         enable = true;
1387         configuration = {
1388           auths.public_v2 = {
1389             community = "public";
1390             version = 2;
1391           };
1392         };
1393       };
1394       exporterTest = ''
1395         wait_for_unit("prometheus-snmp-exporter.service")
1396         wait_for_open_port(9116)
1397         succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1398       '';
1399     };
1400
1401     sql = {
1402       exporterConfig = {
1403         configuration.jobs.points = {
1404           interval = "1m";
1405           connections = [
1406             "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1407           ];
1408           queries = {
1409             points = {
1410               labels = [ "name" ];
1411               help = "Amount of points accumulated per person";
1412               values = [ "amount" ];
1413               query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1414             };
1415           };
1416         };
1417         enable = true;
1418         user = "prometheus-sql-exporter";
1419       };
1420       metricProvider = {
1421         services.postgresql = {
1422           enable = true;
1423           initialScript = builtins.toFile "init.sql" ''
1424             CREATE DATABASE data;
1425             \c data;
1426             CREATE TABLE points (amount INT, name TEXT);
1427             INSERT INTO points(amount, name) VALUES (1, 'jack');
1428             INSERT INTO points(amount, name) VALUES (2, 'jill');
1429             INSERT INTO points(amount, name) VALUES (3, 'jack');
1430
1431             CREATE USER "prometheus-sql-exporter";
1432             GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1433             GRANT SELECT ON points TO "prometheus-sql-exporter";
1434           '';
1435         };
1436         systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1437       };
1438       exporterTest = ''
1439         wait_for_unit("prometheus-sql-exporter.service")
1440         wait_for_open_port(9237)
1441         succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1442       '';
1443     };
1444
1445     statsd = {
1446       exporterConfig = {
1447         enable = true;
1448       };
1449       exporterTest = ''
1450         wait_for_unit("prometheus-statsd-exporter.service")
1451         wait_for_open_port(9102)
1452         succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
1453         wait_until_succeeds(
1454           "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
1455           curl http://localhost:9102/metrics | grep 'test_udp 1'",
1456           timeout=10
1457         )
1458         wait_until_succeeds(
1459           "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
1460           curl http://localhost:9102/metrics | grep 'test_tcp 1'",
1461           timeout=10
1462         )
1463       '';
1464     };
1465
1466     surfboard = {
1467       exporterConfig = {
1468         enable = true;
1469         modemAddress = "localhost";
1470       };
1471       metricProvider = {
1472         systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1473         services.nginx = {
1474           enable = true;
1475           virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1476             return 204;
1477           '';
1478         };
1479       };
1480       exporterTest = ''
1481         wait_for_unit("nginx.service")
1482         wait_for_open_port(80)
1483         wait_for_unit("prometheus-surfboard-exporter.service")
1484         wait_for_open_port(9239)
1485         succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1486       '';
1487     };
1488
1489     systemd = {
1490       exporterConfig = {
1491         enable = true;
1492
1493         extraFlags = [
1494           "--systemd.collector.enable-restart-count"
1495         ];
1496       };
1497       metricProvider = { };
1498       exporterTest = ''
1499         wait_for_unit("prometheus-systemd-exporter.service")
1500         wait_for_open_port(9558)
1501         wait_until_succeeds(
1502             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1503                 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1504             )
1505         )
1506         succeed(
1507             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1508                 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1509             )
1510         )
1511       '';
1512     };
1513
1514     tor = {
1515       exporterConfig = {
1516         enable = true;
1517       };
1518       metricProvider = {
1519         # Note: this does not connect the test environment to the Tor network.
1520         # Client, relay, bridge or exit connectivity are disabled by default.
1521         services.tor.enable = true;
1522         services.tor.settings.ControlPort = 9051;
1523       };
1524       exporterTest = ''
1525         wait_for_unit("tor.service")
1526         wait_for_open_port(9051)
1527         wait_for_unit("prometheus-tor-exporter.service")
1528         wait_for_open_port(9130)
1529         succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
1530       '';
1531     };
1532
1533     unpoller = {
1534       nodeName = "unpoller";
1535       exporterConfig.enable = true;
1536       exporterConfig.controllers = [{ }];
1537       exporterTest = ''
1538         wait_until_succeeds(
1539             'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
1540         )
1541       '';
1542     };
1543
1544     unbound = {
1545       exporterConfig = {
1546         enable = true;
1547         unbound.host = "unix:///run/unbound/unbound.ctl";
1548       };
1549       metricProvider = {
1550         services.unbound = {
1551           enable = true;
1552           localControlSocketPath = "/run/unbound/unbound.ctl";
1553         };
1554         systemd.services.prometheus-unbound-exporter.serviceConfig = {
1555           SupplementaryGroups = [ "unbound" ];
1556         };
1557       };
1558       exporterTest = ''
1559         wait_for_unit("unbound.service")
1560         wait_for_unit("prometheus-unbound-exporter.service")
1561         wait_for_open_port(9167)
1562         wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1563       '';
1564     };
1565
1566     v2ray = {
1567       exporterConfig = {
1568         enable = true;
1569       };
1570
1571       metricProvider = {
1572         systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1573         services.v2ray = {
1574           enable = true;
1575           config = {
1576             stats = {};
1577             api = {
1578               tag = "api";
1579               services = [ "StatsService" ];
1580             };
1581             inbounds = [
1582               {
1583                 port = 1080;
1584                 listen = "127.0.0.1";
1585                 protocol = "http";
1586               }
1587               {
1588                 listen = "127.0.0.1";
1589                 port = 54321;
1590                 protocol = "dokodemo-door";
1591                 settings = { address = "127.0.0.1"; };
1592                 tag = "api";
1593               }
1594             ];
1595             outbounds = [
1596               {
1597                 protocol = "freedom";
1598               }
1599               {
1600                 protocol = "freedom";
1601                 settings = {};
1602                 tag = "api";
1603               }
1604             ];
1605             routing = {
1606               strategy = "rules";
1607               settings = {
1608                 rules = [
1609                   {
1610                     inboundTag = [ "api" ];
1611                     outboundTag = "api";
1612                     type = "field";
1613                   }
1614                 ];
1615               };
1616             };
1617           };
1618         };
1619       };
1620       exporterTest = ''
1621         wait_for_unit("prometheus-v2ray-exporter.service")
1622         wait_for_open_port(9299)
1623         succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1624       '';
1625     };
1626
1627     varnish = {
1628       exporterConfig = {
1629         enable = true;
1630         instance = "/var/spool/varnish/varnish";
1631         group = "varnish";
1632       };
1633       metricProvider = {
1634         systemd.services.prometheus-varnish-exporter.after = [
1635           "varnish.service"
1636         ];
1637         services.varnish = {
1638           enable = true;
1639           config = ''
1640             vcl 4.0;
1641             backend default {
1642               .host = "127.0.0.1";
1643               .port = "80";
1644             }
1645           '';
1646         };
1647       };
1648       exporterTest = ''
1649         wait_for_unit("prometheus-varnish-exporter.service")
1650         wait_for_open_port(6081)
1651         wait_for_open_port(9131)
1652         succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1653       '';
1654     };
1655
1656     wireguard = let
1657       snakeoil = import ./wireguard/snakeoil-keys.nix;
1658       publicKeyWithoutNewlines = replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
1659     in
1660       {
1661         exporterConfig.enable = true;
1662         metricProvider = {
1663           networking.wireguard.interfaces.wg0 = {
1664             ips = [ "10.23.42.1/32" "fc00::1/128" ];
1665             listenPort = 23542;
1666
1667             inherit (snakeoil.peer0) privateKey;
1668
1669             peers = singleton {
1670               allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1671
1672               inherit (snakeoil.peer1) publicKey;
1673             };
1674           };
1675           systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1676         };
1677         exporterTest = ''
1678           wait_for_unit("prometheus-wireguard-exporter.service")
1679           wait_for_open_port(9586)
1680           wait_until_succeeds(
1681               "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
1682           )
1683         '';
1684       };
1685
1686     zfs = {
1687       exporterConfig = {
1688         enable = true;
1689       };
1690       metricProvider = {
1691         boot.supportedFilesystems = [ "zfs" ];
1692         networking.hostId = "7327ded7";
1693       };
1694       exporterTest = ''
1695         wait_for_unit("prometheus-zfs-exporter.service")
1696         wait_for_unit("zfs.target")
1697         wait_for_open_port(9134)
1698         wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
1699       '';
1700     };
1701   };
1702 in
1703 mapAttrs
1704   (exporter: testConfig: (makeTest (
1705     let
1706       nodeName = testConfig.nodeName or exporter;
1707
1708     in
1709     {
1710       name = "prometheus-${exporter}-exporter";
1711
1712       nodes.${nodeName} = mkMerge [{
1713         services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1714       } testConfig.metricProvider or { }];
1715
1716       testScript = ''
1717         ${nodeName}.start()
1718         ${concatStringsSep "\n" (map (line:
1719           if builtins.any (b: b) [
1720             (builtins.match "^[[:space:]]*$" line != null)
1721             (builtins.substring 0 1 line == "#")
1722             (builtins.substring 0 1 line == " ")
1723             (builtins.substring 0 1 line == ")")
1724           ]
1725           then line
1726           else "${nodeName}.${line}"
1727         ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1728         ${nodeName}.shutdown()
1729       '';
1730
1731       meta = with maintainers; {
1732         maintainers = [ willibutz ];
1733       };
1734     }
1735   )))
1736   exporterTests