nixos/tests/prometheus-exporters.nix

   1 { system ? builtins.currentSystem
   2 , config ? { }
   3 , pkgs ? import ../.. { inherit system config; }
   4 }:
   5
   6 let
   7   inherit (import ../lib/testing-python.nix { inherit system pkgs; }) makeTest;
   8   inherit (pkgs.lib) concatStringsSep maintainers mapAttrs mkMerge
   9     removeSuffix replaceChars singleton splitString;
  10
  11   /*
  12     * The attrset `exporterTests` contains one attribute
  13     * for each exporter test. Each of these attributes
  14     * is expected to be an attrset containing:
  15     *
  16     *  `exporterConfig`:
  17     *    this attribute set contains config for the exporter itself
  18     *
  19     *  `exporterTest`
  20     *    this attribute set contains test instructions
  21     *
  22     *  `metricProvider` (optional)
  23     *    this attribute contains additional machine config
  24     *
  25     *  `nodeName` (optional)
  26     *    override an incompatible testnode name
  27     *
  28     *  Example:
  29     *    exporterTests.<exporterName> = {
  30     *      exporterConfig = {
  31     *        enable = true;
  32     *      };
  33     *      metricProvider = {
  34     *        services.<metricProvider>.enable = true;
  35     *      };
  36     *      exporterTest = ''
  37     *        wait_for_unit("prometheus-<exporterName>-exporter.service")
  38     *        wait_for_open_port(1234)
  39     *        succeed("curl -sSf 'localhost:1234/metrics'")
  40     *      '';
  41     *    };
  42     *
  43     *  # this would generate the following test config:
  44     *
  45     *    nodes.<exporterName> = {
  46     *      services.prometheus.<exporterName> = {
  47     *        enable = true;
  48     *      };
  49     *      services.<metricProvider>.enable = true;
  50     *    };
  51     *
  52     *    testScript = ''
  53     *      <exporterName>.start()
  54     *      <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
  55     *      <exporterName>.wait_for_open_port(1234)
  56     *      <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
  57     *      <exporterName>.shutdown()
  58     *    '';
  59   */
  60
  61   exporterTests = {
  62     apcupsd = {
  63       exporterConfig = {
  64         enable = true;
  65       };
  66       metricProvider = {
  67         services.apcupsd.enable = true;
  68       };
  69       exporterTest = ''
  70         wait_for_unit("apcupsd.service")
  71         wait_for_open_port(3551)
  72         wait_for_unit("prometheus-apcupsd-exporter.service")
  73         wait_for_open_port(9162)
  74         succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
  75       '';
  76     };
  77
  78     artifactory = {
  79       exporterConfig = {
  80         enable = true;
  81         artiUsername = "artifactory-username";
  82         artiPassword = "artifactory-password";
  83       };
  84       exporterTest = ''
  85         wait_for_unit("prometheus-artifactory-exporter.service")
  86         wait_for_open_port(9531)
  87         succeed(
  88             "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
  89         )
  90       '';
  91     };
  92
  93     bind = {
  94       exporterConfig = {
  95         enable = true;
  96       };
  97       metricProvider = {
  98         services.bind.enable = true;
  99         services.bind.extraConfig = ''
 100           statistics-channels {
 101             inet 127.0.0.1 port 8053 allow { localhost; };
 102           };
 103         '';
 104       };
 105       exporterTest = ''
 106         wait_for_unit("prometheus-bind-exporter.service")
 107         wait_for_open_port(9119)
 108         succeed(
 109             "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
 110         )
 111       '';
 112     };
 113
 114     bird = {
 115       exporterConfig = {
 116         enable = true;
 117       };
 118       metricProvider = {
 119         services.bird2.enable = true;
 120         services.bird2.config = ''
 121           router id 127.0.0.1;
 122
 123           protocol kernel MyObviousTestString {
 124             ipv4 {
 125               import all;
 126               export none;
 127             };
 128           }
 129
 130           protocol device {
 131           }
 132         '';
 133       };
 134       exporterTest = ''
 135         wait_for_unit("prometheus-bird-exporter.service")
 136         wait_for_open_port(9324)
 137         wait_until_succeeds(
 138             "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
 139         )
 140       '';
 141     };
 142
 143     bitcoin = {
 144       exporterConfig = {
 145         enable = true;
 146         rpcUser = "bitcoinrpc";
 147         rpcPasswordFile = pkgs.writeText "password" "hunter2";
 148       };
 149       metricProvider = {
 150         services.bitcoind.default.enable = true;
 151         services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC = "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
 152       };
 153       exporterTest = ''
 154         wait_for_unit("prometheus-bitcoin-exporter.service")
 155         wait_for_unit("bitcoind-default.service")
 156         wait_for_open_port(9332)
 157         succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
 158       '';
 159     };
 160
 161     blackbox = {
 162       exporterConfig = {
 163         enable = true;
 164         configFile = pkgs.writeText "config.yml" (builtins.toJSON {
 165           modules.icmp_v6 = {
 166             prober = "icmp";
 167             icmp.preferred_ip_protocol = "ip6";
 168           };
 169         });
 170       };
 171       exporterTest = ''
 172         wait_for_unit("prometheus-blackbox-exporter.service")
 173         wait_for_open_port(9115)
 174         succeed(
 175             "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
 176         )
 177       '';
 178     };
 179
 180     collectd = {
 181       exporterConfig = {
 182         enable = true;
 183         extraFlags = [ "--web.collectd-push-path /collectd" ];
 184       };
 185       exporterTest = let postData = replaceChars [ "\n" ] [ "" ] ''
 186         [{
 187           "values":[23],
 188           "dstypes":["gauge"],
 189           "type":"gauge",
 190           "interval":1000,
 191           "host":"testhost",
 192           "plugin":"testplugin",
 193           "time":DATE
 194         }]
 195       ''; in
 196         ''
 197           wait_for_unit("prometheus-collectd-exporter.service")
 198           wait_for_open_port(9103)
 199           succeed(
 200               'echo \'${postData}\'> /tmp/data.json'
 201           )
 202           succeed('sed -ie "s DATE $(date +%s) " /tmp/data.json')
 203           succeed(
 204               "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
 205           )
 206           succeed(
 207               "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
 208           )
 209         '';
 210     };
 211
 212     dnsmasq = {
 213       exporterConfig = {
 214         enable = true;
 215         leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
 216       };
 217       metricProvider = {
 218         services.dnsmasq.enable = true;
 219       };
 220       exporterTest = ''
 221         wait_for_unit("prometheus-dnsmasq-exporter.service")
 222         wait_for_open_port(9153)
 223         succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
 224       '';
 225     };
 226
 227     # Access to WHOIS server is required to properly test this exporter, so
 228     # just perform basic sanity check that the exporter is running and returns
 229     # a failure.
 230     domain = {
 231       exporterConfig = {
 232         enable = true;
 233       };
 234       exporterTest = ''
 235         wait_for_unit("prometheus-domain-exporter.service")
 236         wait_for_open_port(9222)
 237         succeed(
 238             "curl -sSf 'http://localhost:9222/probe?target=nixos.org' | grep 'domain_probe_success 0'"
 239         )
 240       '';
 241     };
 242
 243     dovecot = {
 244       exporterConfig = {
 245         enable = true;
 246         scopes = [ "global" ];
 247         socketPath = "/var/run/dovecot2/old-stats";
 248         user = "root"; # <- don't use user root in production
 249       };
 250       metricProvider = {
 251         services.dovecot2.enable = true;
 252       };
 253       exporterTest = ''
 254         wait_for_unit("prometheus-dovecot-exporter.service")
 255         wait_for_open_port(9166)
 256         succeed(
 257             "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
 258         )
 259       '';
 260     };
 261
 262     fastly = {
 263       exporterConfig = {
 264         enable = true;
 265         tokenPath = pkgs.writeText "token" "abc123";
 266       };
 267
 268       # noop: fastly's exporter can't start without first talking to fastly
 269       # see: https://github.com/peterbourgon/fastly-exporter/issues/87
 270       exporterTest = ''
 271         succeed("true");
 272       '';
 273     };
 274
 275     fritzbox = {
 276       # TODO add proper test case
 277       exporterConfig = {
 278         enable = true;
 279       };
 280       exporterTest = ''
 281         wait_for_unit("prometheus-fritzbox-exporter.service")
 282         wait_for_open_port(9133)
 283         succeed(
 284             "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
 285         )
 286       '';
 287     };
 288
 289     influxdb = {
 290       exporterConfig = {
 291         enable = true;
 292         sampleExpiry = "3s";
 293       };
 294       exporterTest = ''
 295         wait_for_unit("prometheus-influxdb-exporter.service")
 296         wait_for_open_port(9122)
 297         succeed(
 298           "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
 299         )
 300         succeed(
 301           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 302         )
 303         execute("sleep 5")
 304         fail(
 305           "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
 306         )
 307       '';
 308     };
 309
 310     ipmi = {
 311       exporterConfig = {
 312         enable = true;
 313       };
 314       exporterTest = ''
 315         wait_for_unit("prometheus-ipmi-exporter.service")
 316         wait_for_open_port(9290)
 317         succeed(
 318           "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
 319         )
 320       '';
 321     };
 322
 323     jitsi = {
 324       exporterConfig = {
 325         enable = true;
 326       };
 327       metricProvider = {
 328         systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
 329         services.jitsi-videobridge = {
 330           enable = true;
 331           apis = [ "colibri" "rest" ];
 332         };
 333       };
 334       exporterTest = ''
 335         wait_for_unit("jitsi-videobridge2.service")
 336         wait_for_open_port(8080)
 337         wait_for_unit("prometheus-jitsi-exporter.service")
 338         wait_for_open_port(9700)
 339         wait_until_succeeds(
 340             'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
 341         )
 342         succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
 343       '';
 344     };
 345
 346     json = {
 347       exporterConfig = {
 348         enable = true;
 349         url = "http://localhost";
 350         configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON {
 351           metrics = [
 352             { name = "json_test_metric"; path = "{ .test }"; }
 353           ];
 354         });
 355       };
 356       metricProvider = {
 357         systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
 358         services.nginx = {
 359           enable = true;
 360           virtualHosts.localhost.locations."/".extraConfig = ''
 361             return 200 "{\"test\":1}";
 362           '';
 363         };
 364       };
 365       exporterTest = ''
 366         wait_for_unit("nginx.service")
 367         wait_for_open_port(80)
 368         wait_for_unit("prometheus-json-exporter.service")
 369         wait_for_open_port(7979)
 370         succeed(
 371             "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
 372         )
 373       '';
 374     };
 375
 376     kea = let
 377       controlSocketPathV4 = "/run/kea/dhcp4.sock";
 378       controlSocketPathV6 = "/run/kea/dhcp6.sock";
 379     in
 380     {
 381       exporterConfig = {
 382         enable = true;
 383         controlSocketPaths = [
 384           controlSocketPathV4
 385           controlSocketPathV6
 386         ];
 387       };
 388       metricProvider = {
 389         services.kea = {
 390           dhcp4 = {
 391             enable = true;
 392             settings = {
 393               control-socket = {
 394                 socket-type = "unix";
 395                 socket-name = controlSocketPathV4;
 396               };
 397             };
 398           };
 399           dhcp6 = {
 400             enable = true;
 401             settings = {
 402               control-socket = {
 403                 socket-type = "unix";
 404                 socket-name = controlSocketPathV6;
 405               };
 406             };
 407           };
 408         };
 409       };
 410
 411       exporterTest = ''
 412         wait_for_unit("kea-dhcp4-server.service")
 413         wait_for_unit("kea-dhcp6-server.service")
 414         wait_for_file("${controlSocketPathV4}")
 415         wait_for_file("${controlSocketPathV6}")
 416         wait_for_unit("prometheus-kea-exporter.service")
 417         wait_for_open_port(9547)
 418         succeed(
 419             "curl --fail localhost:9547/metrics | grep 'packets_received_total'"
 420         )
 421       '';
 422     };
 423
 424     knot = {
 425       exporterConfig = {
 426         enable = true;
 427       };
 428       metricProvider = {
 429         services.knot = {
 430           enable = true;
 431           extraArgs = [ "-v" ];
 432           extraConfig = ''
 433             server:
 434               listen: 127.0.0.1@53
 435
 436             template:
 437               - id: default
 438                 global-module: mod-stats
 439                 dnssec-signing: off
 440                 zonefile-sync: -1
 441                 journal-db: /var/lib/knot/journal
 442                 kasp-db: /var/lib/knot/kasp
 443                 timer-db: /var/lib/knot/timer
 444                 zonefile-load: difference
 445                 storage: ${pkgs.buildEnv {
 446                   name = "foo";
 447                   paths = [
 448                     (pkgs.writeTextDir "test.zone" ''
 449                       @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
 450                       @       NS      ns1
 451                       @       NS      ns2
 452                       ns1     A       192.168.0.1
 453                     '')
 454                   ];
 455                 }}
 456
 457             mod-stats:
 458               - id: custom
 459                 edns-presence: on
 460                 query-type: on
 461
 462             zone:
 463               - domain: test
 464                 file: test.zone
 465                 module: mod-stats/custom
 466           '';
 467         };
 468       };
 469       exporterTest = ''
 470         wait_for_unit("knot.service")
 471         wait_for_unit("prometheus-knot-exporter.service")
 472         wait_for_open_port(9433)
 473         succeed("curl -sSf 'localhost:9433' | grep 'knot_server_zone_count 1.0'")
 474       '';
 475     };
 476
 477     keylight = {
 478       # A hardware device is required to properly test this exporter, so just
 479       # perform a couple of basic sanity checks that the exporter is running
 480       # and requires a target, but cannot reach a specified target.
 481       exporterConfig = {
 482         enable = true;
 483       };
 484       exporterTest = ''
 485         wait_for_unit("prometheus-keylight-exporter.service")
 486         wait_for_open_port(9288)
 487         succeed(
 488             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
 489         )
 490         succeed(
 491             "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
 492         )
 493       '';
 494     };
 495
 496     lnd = {
 497       exporterConfig = {
 498         enable = true;
 499         lndTlsPath = "/var/lib/lnd/tls.cert";
 500         lndMacaroonDir = "/var/lib/lnd";
 501         extraFlags = [ "--lnd.network=regtest" ];
 502       };
 503       metricProvider = {
 504         systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
 505         systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
 506         services.bitcoind.regtest = {
 507           enable = true;
 508           extraConfig = ''
 509             rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
 510             zmqpubrawblock=tcp://127.0.0.1:28332
 511             zmqpubrawtx=tcp://127.0.0.1:28333
 512           '';
 513           extraCmdlineOptions = [ "-regtest" ];
 514         };
 515         systemd.services.lnd = {
 516           serviceConfig.ExecStart = ''
 517             ${pkgs.lnd}/bin/lnd \
 518               --datadir=/var/lib/lnd \
 519               --tlscertpath=/var/lib/lnd/tls.cert \
 520               --tlskeypath=/var/lib/lnd/tls.key \
 521               --logdir=/var/log/lnd \
 522               --bitcoin.active \
 523               --bitcoin.regtest \
 524               --bitcoin.node=bitcoind \
 525               --bitcoind.rpcuser=bitcoinrpc \
 526               --bitcoind.rpcpass=hunter2 \
 527               --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
 528               --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
 529               --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
 530           '';
 531           serviceConfig.StateDirectory = "lnd";
 532           wantedBy = [ "multi-user.target" ];
 533           after = [ "network.target" ];
 534         };
 535         # initialize wallet, creates macaroon needed by exporter
 536         systemd.services.lnd.postStart = ''
 537           ${pkgs.curl}/bin/curl \
 538             --retry 20 \
 539             --retry-delay 1 \
 540             --retry-connrefused \
 541             --cacert /var/lib/lnd/tls.cert \
 542             -X GET \
 543             https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
 544           ${pkgs.curl}/bin/curl \
 545             --retry 20 \
 546             --retry-delay 1 \
 547             --retry-connrefused \
 548             --cacert /var/lib/lnd/tls.cert \
 549             -X POST \
 550             -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
 551             https://localhost:8080/v1/initwallet
 552         '';
 553       };
 554       exporterTest = ''
 555         wait_for_unit("lnd.service")
 556         wait_for_open_port(10009)
 557         wait_for_unit("prometheus-lnd-exporter.service")
 558         wait_for_open_port(9092)
 559         succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
 560       '';
 561     };
 562
 563     mail = {
 564       exporterConfig = {
 565         enable = true;
 566         configuration = {
 567           monitoringInterval = "2s";
 568           mailCheckTimeout = "10s";
 569           servers = [{
 570             name = "testserver";
 571             server = "localhost";
 572             port = 25;
 573             from = "mail-exporter@localhost";
 574             to = "mail-exporter@localhost";
 575             detectionDir = "/var/spool/mail/mail-exporter/new";
 576           }];
 577         };
 578       };
 579       metricProvider = {
 580         services.postfix.enable = true;
 581         systemd.services.prometheus-mail-exporter = {
 582           after = [ "postfix.service" ];
 583           requires = [ "postfix.service" ];
 584           serviceConfig = {
 585             ExecStartPre = [
 586               "${pkgs.writeShellScript "create-maildir" ''
 587                 mkdir -p -m 0700 mail-exporter/new
 588               ''}"
 589             ];
 590             ProtectHome = true;
 591             ReadOnlyPaths = "/";
 592             ReadWritePaths = "/var/spool/mail";
 593             WorkingDirectory = "/var/spool/mail";
 594           };
 595         };
 596         users.users.mailexporter = {
 597           isSystemUser = true;
 598           group = "mailexporter";
 599         };
 600         users.groups.mailexporter = {};
 601       };
 602       exporterTest = ''
 603         wait_for_unit("postfix.service")
 604         wait_for_unit("prometheus-mail-exporter.service")
 605         wait_for_open_port(9225)
 606         wait_until_succeeds(
 607             "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
 608         )
 609       '';
 610     };
 611
 612     mikrotik = {
 613       exporterConfig = {
 614         enable = true;
 615         extraFlags = [ "-timeout=1s" ];
 616         configuration = {
 617           devices = [
 618             {
 619               name = "router";
 620               address = "192.168.42.48";
 621               user = "prometheus";
 622               password = "shh";
 623             }
 624           ];
 625           features = {
 626             bgp = true;
 627             dhcp = true;
 628             dhcpl = true;
 629             dhcpv6 = true;
 630             health = true;
 631             routes = true;
 632             poe = true;
 633             pools = true;
 634             optics = true;
 635             w60g = true;
 636             wlansta = true;
 637             wlanif = true;
 638             monitor = true;
 639             ipsec = true;
 640           };
 641         };
 642       };
 643       exporterTest = ''
 644         wait_for_unit("prometheus-mikrotik-exporter.service")
 645         wait_for_open_port(9436)
 646         succeed(
 647             "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
 648         )
 649       '';
 650     };
 651
 652     modemmanager = {
 653       exporterConfig = {
 654         enable = true;
 655         refreshRate = "10s";
 656       };
 657       metricProvider = {
 658         # ModemManager is installed when NetworkManager is enabled. Ensure it is
 659         # started and is wanted by NM and the exporter to start everything up
 660         # in the right order.
 661         networking.networkmanager.enable = true;
 662         systemd.services.ModemManager = {
 663           enable = true;
 664           wantedBy = [ "NetworkManager.service" "prometheus-modemmanager-exporter.service" ];
 665         };
 666       };
 667       exporterTest = ''
 668         wait_for_unit("ModemManager.service")
 669         wait_for_unit("prometheus-modemmanager-exporter.service")
 670         wait_for_open_port(9539)
 671         succeed(
 672             "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
 673         )
 674       '';
 675     };
 676
 677     nextcloud = {
 678       exporterConfig = {
 679         enable = true;
 680         passwordFile = "/var/nextcloud-pwfile";
 681         url = "http://localhost";
 682       };
 683       metricProvider = {
 684         systemd.services.nc-pwfile =
 685           let
 686             passfile = (pkgs.writeText "pwfile" "snakeoilpw");
 687           in
 688           {
 689             requiredBy = [ "prometheus-nextcloud-exporter.service" ];
 690             before = [ "prometheus-nextcloud-exporter.service" ];
 691             serviceConfig.ExecStart = ''
 692               ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
 693             '';
 694           };
 695         services.nginx = {
 696           enable = true;
 697           virtualHosts."localhost" = {
 698             basicAuth.nextcloud-exporter = "snakeoilpw";
 699             locations."/" = {
 700               root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
 701               tryFiles = "/negative-space.json =404";
 702             };
 703           };
 704         };
 705       };
 706       exporterTest = ''
 707         wait_for_unit("nginx.service")
 708         wait_for_unit("prometheus-nextcloud-exporter.service")
 709         wait_for_open_port(9205)
 710         succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
 711       '';
 712     };
 713
 714     nginx = {
 715       exporterConfig = {
 716         enable = true;
 717       };
 718       metricProvider = {
 719         services.nginx = {
 720           enable = true;
 721           statusPage = true;
 722           virtualHosts."test".extraConfig = "return 204;";
 723         };
 724       };
 725       exporterTest = ''
 726         wait_for_unit("nginx.service")
 727         wait_for_unit("prometheus-nginx-exporter.service")
 728         wait_for_open_port(9113)
 729         succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up 1'")
 730       '';
 731     };
 732
 733     nginxlog = {
 734       exporterConfig = {
 735         enable = true;
 736         group = "nginx";
 737         settings = {
 738           namespaces = [
 739             {
 740               name = "filelogger";
 741               source = {
 742                 files = [ "/var/log/nginx/filelogger.access.log" ];
 743               };
 744             }
 745             {
 746               name = "syslogger";
 747               source = {
 748                 syslog = {
 749                   listen_address = "udp://127.0.0.1:10000";
 750                   format = "rfc3164";
 751                   tags = [ "nginx" ];
 752                 };
 753               };
 754             }
 755           ];
 756         };
 757       };
 758       metricProvider = {
 759         services.nginx = {
 760           enable = true;
 761           httpConfig = ''
 762             server {
 763               listen 80;
 764               server_name filelogger.local;
 765               access_log /var/log/nginx/filelogger.access.log;
 766             }
 767             server {
 768               listen 81;
 769               server_name syslogger.local;
 770               access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
 771             }
 772           '';
 773         };
 774       };
 775       exporterTest = ''
 776         wait_for_unit("nginx.service")
 777         wait_for_unit("prometheus-nginxlog-exporter.service")
 778         wait_for_open_port(9117)
 779         wait_for_open_port(80)
 780         wait_for_open_port(81)
 781         succeed("curl http://localhost")
 782         execute("sleep 1")
 783         succeed(
 784             "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
 785         )
 786         succeed("curl http://localhost:81")
 787         execute("sleep 1")
 788         succeed(
 789             "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
 790         )
 791       '';
 792     };
 793
 794     node = {
 795       exporterConfig = {
 796         enable = true;
 797       };
 798       exporterTest = ''
 799         wait_for_unit("prometheus-node-exporter.service")
 800         wait_for_open_port(9100)
 801         succeed(
 802             "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
 803         )
 804       '';
 805     };
 806
 807     openldap = {
 808       exporterConfig = {
 809         enable = true;
 810         ldapCredentialFile = "${pkgs.writeText "exporter.yml" ''
 811           ldapUser: "cn=root,dc=example"
 812           ldapPass: "notapassword"
 813         ''}";
 814       };
 815       metricProvider = {
 816         services.openldap = {
 817           enable = true;
 818           settings.children = {
 819             "cn=schema".includes = [
 820               "${pkgs.openldap}/etc/schema/core.ldif"
 821               "${pkgs.openldap}/etc/schema/cosine.ldif"
 822               "${pkgs.openldap}/etc/schema/inetorgperson.ldif"
 823               "${pkgs.openldap}/etc/schema/nis.ldif"
 824             ];
 825             "olcDatabase={1}mdb" = {
 826               attrs = {
 827                 objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ];
 828                 olcDatabase = "{1}mdb";
 829                 olcDbDirectory = "/var/db/openldap";
 830                 olcSuffix = "dc=example";
 831                 olcRootDN = {
 832                   # cn=root,dc=example
 833                   base64 = "Y249cm9vdCxkYz1leGFtcGxl";
 834                 };
 835                 olcRootPW = {
 836                   path = "${pkgs.writeText "rootpw" "notapassword"}";
 837                 };
 838               };
 839             };
 840             "olcDatabase={2}monitor".attrs = {
 841               objectClass = [ "olcDatabaseConfig" ];
 842               olcDatabase = "{2}monitor";
 843               olcAccess = [ "to dn.subtree=cn=monitor by users read" ];
 844             };
 845           };
 846           declarativeContents."dc=example" = ''
 847             dn: dc=example
 848             objectClass: domain
 849             dc: example
 850
 851             dn: ou=users,dc=example
 852             objectClass: organizationalUnit
 853             ou: users
 854           '';
 855         };
 856       };
 857       exporterTest = ''
 858         wait_for_unit("prometheus-openldap-exporter.service")
 859         wait_for_open_port(389)
 860         wait_for_open_port(9330)
 861         wait_until_succeeds(
 862             "curl -sSf http://localhost:9330/metrics | grep 'openldap_scrape{result=\"ok\"} 1'"
 863         )
 864       '';
 865     };
 866
 867     openvpn = {
 868       exporterConfig = {
 869         enable = true;
 870         group = "openvpn";
 871         statusPaths = [ "/run/openvpn-test" ];
 872       };
 873       metricProvider = {
 874         users.groups.openvpn = { };
 875         services.openvpn.servers.test = {
 876           config = ''
 877             dev tun
 878             status /run/openvpn-test
 879             status-version 3
 880           '';
 881           up = "chmod g+r /run/openvpn-test";
 882         };
 883         systemd.services."openvpn-test".serviceConfig.Group = "openvpn";
 884       };
 885       exporterTest = ''
 886         wait_for_unit("openvpn-test.service")
 887         wait_for_unit("prometheus-openvpn-exporter.service")
 888         succeed("curl -sSf http://localhost:9176/metrics | grep 'openvpn_up{.*} 1'")
 889       '';
 890     };
 891
 892     postfix = {
 893       exporterConfig = {
 894         enable = true;
 895       };
 896       metricProvider = {
 897         services.postfix.enable = true;
 898       };
 899       exporterTest = ''
 900         wait_for_unit("prometheus-postfix-exporter.service")
 901         wait_for_file("/var/lib/postfix/queue/public/showq")
 902         wait_for_open_port(9154)
 903         wait_until_succeeds(
 904             "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"/var/lib/postfix/queue/public/showq\"} 1'"
 905         )
 906         succeed(
 907             "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
 908         )
 909         succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
 910       '';
 911     };
 912
 913     postgres = {
 914       exporterConfig = {
 915         enable = true;
 916         runAsLocalSuperUser = true;
 917       };
 918       metricProvider = {
 919         services.postgresql.enable = true;
 920       };
 921       exporterTest = ''
 922         wait_for_unit("prometheus-postgres-exporter.service")
 923         wait_for_open_port(9187)
 924         wait_for_unit("postgresql.service")
 925         succeed(
 926             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
 927         )
 928         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
 929         systemctl("stop postgresql.service")
 930         succeed(
 931             "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
 932         )
 933         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
 934         systemctl("start postgresql.service")
 935         wait_for_unit("postgresql.service")
 936         succeed(
 937             "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
 938         )
 939         succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
 940       '';
 941     };
 942
 943     process = {
 944       exporterConfig = {
 945         enable = true;
 946         settings.process_names = [
 947           # Remove nix store path from process name
 948           { name = "{{.Matches.Wrapped}} {{ .Matches.Args }}"; cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ]; }
 949         ];
 950       };
 951       exporterTest = ''
 952         wait_for_unit("prometheus-process-exporter.service")
 953         wait_for_open_port(9256)
 954         wait_until_succeeds(
 955             "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
 956                 'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
 957             )
 958         )
 959       '';
 960     };
 961
 962     pve = let
 963       pveExporterEnvFile = pkgs.writeTextFile {
 964         name = "pve.env";
 965         text = ''
 966           PVE_USER="test_user@pam"
 967           PVE_PASSWORD="hunter3"
 968           PVE_VERIFY_SSL="false"
 969         '';
 970       };
 971     in {
 972       exporterConfig = {
 973         enable = true;
 974         environmentFile = pveExporterEnvFile;
 975       };
 976       exporterTest = ''
 977         wait_for_unit("prometheus-pve-exporter.service")
 978         wait_for_open_port(9221)
 979         wait_until_succeeds("curl localhost:9221")
 980       '';
 981     };
 982
 983     py-air-control = {
 984       nodeName = "py_air_control";
 985       exporterConfig = {
 986         enable = true;
 987         deviceHostname = "127.0.0.1";
 988       };
 989       exporterTest = ''
 990         wait_for_unit("prometheus-py-air-control-exporter.service")
 991         wait_for_open_port(9896)
 992         succeed(
 993             "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
 994         )
 995       '';
 996     };
 997
 998     redis = {
 999       exporterConfig = {
1000         enable = true;
1001       };
1002       metricProvider.services.redis.servers."".enable = true;
1003       exporterTest = ''
1004         wait_for_unit("redis.service")
1005         wait_for_unit("prometheus-redis-exporter.service")
1006         wait_for_open_port(6379)
1007         wait_for_open_port(9121)
1008         wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
1009       '';
1010     };
1011
1012     rspamd = {
1013       exporterConfig = {
1014         enable = true;
1015       };
1016       metricProvider = {
1017         services.rspamd.enable = true;
1018       };
1019       exporterTest = ''
1020         wait_for_unit("rspamd.service")
1021         wait_for_unit("prometheus-rspamd-exporter.service")
1022         wait_for_open_port(11334)
1023         wait_for_open_port(7980)
1024         wait_until_succeeds(
1025             "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep 'rspamd_scanned{host=\"rspamd\"} 0'"
1026         )
1027       '';
1028     };
1029
1030     rtl_433 = {
1031       exporterConfig = {
1032         enable = true;
1033       };
1034       metricProvider = {
1035         # Mock rtl_433 binary to return a dummy metric stream.
1036         nixpkgs.overlays = [
1037           (self: super: {
1038             rtl_433 = self.runCommand "rtl_433" { } ''
1039               mkdir -p "$out/bin"
1040               cat <<EOF > "$out/bin/rtl_433"
1041               #!/bin/sh
1042               while true; do
1043                 printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
1044                 sleep 4
1045               done
1046               EOF
1047               chmod +x "$out/bin/rtl_433"
1048             '';
1049           })
1050         ];
1051       };
1052       exporterTest = ''
1053         wait_for_unit("prometheus-rtl_433-exporter.service")
1054         wait_for_open_port(9550)
1055         wait_until_succeeds(
1056             "curl -sSf localhost:9550/metrics | grep '{}'".format(
1057                 'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
1058             )
1059         )
1060       '';
1061     };
1062
1063     script = {
1064       exporterConfig = {
1065         enable = true;
1066         settings.scripts = [
1067           { name = "success"; script = "sleep 1"; }
1068         ];
1069       };
1070       exporterTest = ''
1071         wait_for_unit("prometheus-script-exporter.service")
1072         wait_for_open_port(9172)
1073         wait_until_succeeds(
1074             "curl -sSf 'localhost:9172/probe?name=success' | grep -q '{}'".format(
1075                 'script_success{script="success"} 1'
1076             )
1077         )
1078       '';
1079     };
1080
1081     smartctl = {
1082       exporterConfig = {
1083         enable = true;
1084         devices = [
1085           "/dev/vda"
1086         ];
1087       };
1088       exporterTest = ''
1089         wait_for_unit("prometheus-smartctl-exporter.service")
1090         wait_for_open_port(9633)
1091         wait_until_succeeds(
1092           "curl -sSf 'localhost:9633/metrics'"
1093         )
1094         wait_until_succeeds(
1095             'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "/dev/vda: Unable to detect device type"'
1096         )
1097       '';
1098     };
1099
1100     smokeping = {
1101       exporterConfig = {
1102         enable = true;
1103         hosts = [ "127.0.0.1" ];
1104       };
1105       exporterTest = ''
1106         wait_for_unit("prometheus-smokeping-exporter.service")
1107         wait_for_open_port(9374)
1108         wait_until_succeeds(
1109             "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
1110                 'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1"} '
1111             )
1112         )
1113         wait_until_succeeds(
1114             "curl -sSf localhost:9374/metrics | grep '{}'".format(
1115                 'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1"}'
1116             )
1117         )
1118       '';
1119     };
1120
1121     snmp = {
1122       exporterConfig = {
1123         enable = true;
1124         configuration.default = {
1125           version = 2;
1126           auth.community = "public";
1127         };
1128       };
1129       exporterTest = ''
1130         wait_for_unit("prometheus-snmp-exporter.service")
1131         wait_for_open_port(9116)
1132         succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
1133       '';
1134     };
1135
1136     sql = {
1137       exporterConfig = {
1138         configuration.jobs.points = {
1139           interval = "1m";
1140           connections = [
1141             "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
1142           ];
1143           queries = {
1144             points = {
1145               labels = [ "name" ];
1146               help = "Amount of points accumulated per person";
1147               values = [ "amount" ];
1148               query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
1149             };
1150           };
1151         };
1152         enable = true;
1153         user = "prometheus-sql-exporter";
1154       };
1155       metricProvider = {
1156         services.postgresql = {
1157           enable = true;
1158           initialScript = builtins.toFile "init.sql" ''
1159             CREATE DATABASE data;
1160             \c data;
1161             CREATE TABLE points (amount INT, name TEXT);
1162             INSERT INTO points(amount, name) VALUES (1, 'jack');
1163             INSERT INTO points(amount, name) VALUES (2, 'jill');
1164             INSERT INTO points(amount, name) VALUES (3, 'jack');
1165
1166             CREATE USER "prometheus-sql-exporter";
1167             GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
1168             GRANT SELECT ON points TO "prometheus-sql-exporter";
1169           '';
1170         };
1171         systemd.services.prometheus-sql-exporter.after = [ "postgresql.service" ];
1172       };
1173       exporterTest = ''
1174         wait_for_unit("prometheus-sql-exporter.service")
1175         wait_for_open_port(9237)
1176         succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
1177       '';
1178     };
1179
1180     surfboard = {
1181       exporterConfig = {
1182         enable = true;
1183         modemAddress = "localhost";
1184       };
1185       metricProvider = {
1186         systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
1187         services.nginx = {
1188           enable = true;
1189           virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
1190             return 204;
1191           '';
1192         };
1193       };
1194       exporterTest = ''
1195         wait_for_unit("nginx.service")
1196         wait_for_open_port(80)
1197         wait_for_unit("prometheus-surfboard-exporter.service")
1198         wait_for_open_port(9239)
1199         succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
1200       '';
1201     };
1202
1203     systemd = {
1204       exporterConfig = {
1205         enable = true;
1206
1207         extraFlags = [
1208           "--systemd.collector.enable-restart-count"
1209         ];
1210       };
1211       metricProvider = { };
1212       exporterTest = ''
1213         wait_for_unit("prometheus-systemd-exporter.service")
1214         wait_for_open_port(9558)
1215         wait_until_succeeds(
1216             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1217                 'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
1218             )
1219         )
1220         succeed(
1221             "curl -sSf localhost:9558/metrics | grep '{}'".format(
1222                 'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
1223             )
1224         )
1225       '';
1226     };
1227
1228     tor = {
1229       exporterConfig = {
1230         enable = true;
1231       };
1232       metricProvider = {
1233         # Note: this does not connect the test environment to the Tor network.
1234         # Client, relay, bridge or exit connectivity are disabled by default.
1235         services.tor.enable = true;
1236         services.tor.settings.ControlPort = 9051;
1237       };
1238       exporterTest = ''
1239         wait_for_unit("tor.service")
1240         wait_for_open_port(9051)
1241         wait_for_unit("prometheus-tor-exporter.service")
1242         wait_for_open_port(9130)
1243         succeed("curl -sSf localhost:9130/metrics | grep 'tor_version{.\\+} 1'")
1244       '';
1245     };
1246
1247     unifi-poller = {
1248       nodeName = "unifi_poller";
1249       exporterConfig.enable = true;
1250       exporterConfig.controllers = [{ }];
1251       exporterTest = ''
1252         wait_for_unit("prometheus-unifi-poller-exporter.service")
1253         wait_for_open_port(9130)
1254         succeed(
1255             "curl -sSf localhost:9130/metrics | grep 'unifipoller_build_info{.\\+} 1'"
1256         )
1257       '';
1258     };
1259
1260     unbound = {
1261       exporterConfig = {
1262         enable = true;
1263         fetchType = "uds";
1264         controlInterface = "/run/unbound/unbound.ctl";
1265       };
1266       metricProvider = {
1267         services.unbound = {
1268           enable = true;
1269           localControlSocketPath = "/run/unbound/unbound.ctl";
1270         };
1271         systemd.services.prometheus-unbound-exporter.serviceConfig = {
1272           SupplementaryGroups = [ "unbound" ];
1273         };
1274       };
1275       exporterTest = ''
1276         wait_for_unit("unbound.service")
1277         wait_for_unit("prometheus-unbound-exporter.service")
1278         wait_for_open_port(9167)
1279         succeed("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
1280       '';
1281     };
1282
1283     v2ray = {
1284       exporterConfig = {
1285         enable = true;
1286       };
1287
1288       metricProvider = {
1289         systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
1290         services.v2ray = {
1291           enable = true;
1292           config = {
1293             stats = {};
1294             api = {
1295               tag = "api";
1296               services = [ "StatsService" ];
1297             };
1298             inbounds = [
1299               {
1300                 port = 1080;
1301                 listen = "127.0.0.1";
1302                 protocol = "http";
1303               }
1304               {
1305                 listen = "127.0.0.1";
1306                 port = 54321;
1307                 protocol = "dokodemo-door";
1308                 settings = { address = "127.0.0.1"; };
1309                 tag = "api";
1310               }
1311             ];
1312             outbounds = [
1313               {
1314                 protocol = "freedom";
1315               }
1316               {
1317                 protocol = "freedom";
1318                 settings = {};
1319                 tag = "api";
1320               }
1321             ];
1322             routing = {
1323               strategy = "rules";
1324               settings = {
1325                 rules = [
1326                   {
1327                     inboundTag = [ "api" ];
1328                     outboundTag = "api";
1329                     type = "field";
1330                   }
1331                 ];
1332               };
1333             };
1334           };
1335         };
1336       };
1337       exporterTest = ''
1338         wait_for_unit("prometheus-v2ray-exporter.service")
1339         wait_for_open_port(9299)
1340         succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
1341       '';
1342     };
1343
1344     varnish = {
1345       exporterConfig = {
1346         enable = true;
1347         instance = "/var/spool/varnish/varnish";
1348         group = "varnish";
1349       };
1350       metricProvider = {
1351         systemd.services.prometheus-varnish-exporter.after = [
1352           "varnish.service"
1353         ];
1354         services.varnish = {
1355           enable = true;
1356           config = ''
1357             vcl 4.0;
1358             backend default {
1359               .host = "127.0.0.1";
1360               .port = "80";
1361             }
1362           '';
1363         };
1364       };
1365       exporterTest = ''
1366         wait_for_unit("prometheus-varnish-exporter.service")
1367         wait_for_open_port(6081)
1368         wait_for_open_port(9131)
1369         succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
1370       '';
1371     };
1372
1373     wireguard = let snakeoil = import ./wireguard/snakeoil-keys.nix; in
1374       {
1375         exporterConfig.enable = true;
1376         metricProvider = {
1377           networking.wireguard.interfaces.wg0 = {
1378             ips = [ "10.23.42.1/32" "fc00::1/128" ];
1379             listenPort = 23542;
1380
1381             inherit (snakeoil.peer0) privateKey;
1382
1383             peers = singleton {
1384               allowedIPs = [ "10.23.42.2/32" "fc00::2/128" ];
1385
1386               inherit (snakeoil.peer1) publicKey;
1387             };
1388           };
1389           systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
1390         };
1391         exporterTest = ''
1392           wait_for_unit("prometheus-wireguard-exporter.service")
1393           wait_for_open_port(9586)
1394           wait_until_succeeds(
1395               "curl -sSf http://localhost:9586/metrics | grep '${snakeoil.peer1.publicKey}'"
1396           )
1397         '';
1398       };
1399   };
1400 in
1401 mapAttrs
1402   (exporter: testConfig: (makeTest (
1403     let
1404       nodeName = testConfig.nodeName or exporter;
1405
1406     in
1407     {
1408       name = "prometheus-${exporter}-exporter";
1409
1410       nodes.${nodeName} = mkMerge [{
1411         services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
1412       } testConfig.metricProvider or { }];
1413
1414       testScript = ''
1415         ${nodeName}.start()
1416         ${concatStringsSep "\n" (map (line:
1417           if (builtins.substring 0 1 line == " " || builtins.substring 0 1 line == ")")
1418           then line
1419           else "${nodeName}.${line}"
1420         ) (splitString "\n" (removeSuffix "\n" testConfig.exporterTest)))}
1421         ${nodeName}.shutdown()
1422       '';
1423
1424       meta = with maintainers; {
1425         maintainers = [ willibutz ];
1426       };
1427     }
1428   )))
1429   exporterTests