nixos/tests/prometheus/federation.nix

   1 import ../make-test-python.nix ({ lib, pkgs, ... }:
   2
   3 {
   4   name = "prometheus-federation";
   5
   6   nodes = {
   7     global1 = { config, pkgs, ... }: {
   8       environment.systemPackages = [ pkgs.jq ];
   9
  10       networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
  11
  12       services.prometheus = {
  13         enable = true;
  14         globalConfig.scrape_interval = "2s";
  15
  16         scrapeConfigs = [
  17           {
  18             job_name = "federate";
  19             honor_labels = true;
  20             metrics_path = "/federate";
  21
  22             params = {
  23               "match[]" = [
  24                 "{job=\"node\"}"
  25                 "{job=\"prometheus\"}"
  26               ];
  27             };
  28
  29             static_configs = [
  30               {
  31                 targets = [
  32                   "prometheus1:${toString config.services.prometheus.port}"
  33                   "prometheus2:${toString config.services.prometheus.port}"
  34                 ];
  35               }
  36             ];
  37           }
  38           {
  39             job_name = "prometheus";
  40             static_configs = [
  41               {
  42                 targets = [
  43                   "global1:${toString config.services.prometheus.port}"
  44                   "global2:${toString config.services.prometheus.port}"
  45                 ];
  46               }
  47             ];
  48           }
  49         ];
  50       };
  51     };
  52
  53     global2 = { config, pkgs, ... }: {
  54       environment.systemPackages = [ pkgs.jq ];
  55
  56       networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
  57
  58       services.prometheus = {
  59         enable = true;
  60         globalConfig.scrape_interval = "2s";
  61
  62         scrapeConfigs = [
  63           {
  64             job_name = "federate";
  65             honor_labels = true;
  66             metrics_path = "/federate";
  67
  68             params = {
  69               "match[]" = [
  70                 "{job=\"node\"}"
  71                 "{job=\"prometheus\"}"
  72               ];
  73             };
  74
  75             static_configs = [
  76               {
  77                 targets = [
  78                   "prometheus1:${toString config.services.prometheus.port}"
  79                   "prometheus2:${toString config.services.prometheus.port}"
  80                 ];
  81               }
  82             ];
  83           }
  84           {
  85             job_name = "prometheus";
  86             static_configs = [
  87               {
  88                 targets = [
  89                   "global1:${toString config.services.prometheus.port}"
  90                   "global2:${toString config.services.prometheus.port}"
  91                 ];
  92               }
  93             ];
  94           }
  95         ];
  96       };
  97     };
  98
  99     prometheus1 = { config, pkgs, ... }: {
 100       environment.systemPackages = [ pkgs.jq ];
 101
 102       networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 103
 104       services.prometheus = {
 105         enable = true;
 106         globalConfig.scrape_interval = "2s";
 107
 108         scrapeConfigs = [
 109           {
 110             job_name = "node";
 111             static_configs = [
 112               {
 113                 targets = [
 114                   "node1:${toString config.services.prometheus.exporters.node.port}"
 115                 ];
 116               }
 117             ];
 118           }
 119           {
 120             job_name = "prometheus";
 121             static_configs = [
 122               {
 123                 targets = [
 124                   "prometheus1:${toString config.services.prometheus.port}"
 125                 ];
 126               }
 127             ];
 128           }
 129         ];
 130       };
 131     };
 132
 133     prometheus2 = { config, pkgs, ... }: {
 134       environment.systemPackages = [ pkgs.jq ];
 135
 136       networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 137
 138       services.prometheus = {
 139         enable = true;
 140         globalConfig.scrape_interval = "2s";
 141
 142         scrapeConfigs = [
 143           {
 144             job_name = "node";
 145             static_configs = [
 146               {
 147                 targets = [
 148                   "node2:${toString config.services.prometheus.exporters.node.port}"
 149                 ];
 150               }
 151             ];
 152           }
 153           {
 154             job_name = "prometheus";
 155             static_configs = [
 156               {
 157                 targets = [
 158                   "prometheus2:${toString config.services.prometheus.port}"
 159                 ];
 160               }
 161             ];
 162           }
 163         ];
 164       };
 165     };
 166
 167     node1 = { config, pkgs, ... }: {
 168       services.prometheus.exporters.node = {
 169         enable = true;
 170         openFirewall = true;
 171       };
 172     };
 173
 174     node2 = { config, pkgs, ... }: {
 175       services.prometheus.exporters.node = {
 176         enable = true;
 177         openFirewall = true;
 178       };
 179     };
 180   };
 181
 182   testScript = ''
 183     for machine in node1, node2:
 184       machine.wait_for_unit("prometheus-node-exporter")
 185       machine.wait_for_open_port(9100)
 186
 187     for machine in prometheus1, prometheus2, global1, global2:
 188       machine.wait_for_unit("prometheus")
 189       machine.wait_for_open_port(9090)
 190
 191     # Verify both servers got the same data from the exporter
 192     for machine in prometheus1, prometheus2:
 193       machine.wait_until_succeeds(
 194         "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
 195         + "jq '.data.result[0].value[1]' | grep '\"1\"'"
 196       )
 197       machine.wait_until_succeeds(
 198         "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
 199         + "jq '.data.result[0].value[1]' | grep '\"1\"'"
 200       )
 201
 202     for machine in global1, global2:
 203       machine.wait_until_succeeds(
 204         "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
 205         + "jq '.data.result[0].value[1]' | grep '\"2\"'"
 206       )
 207
 208       machine.wait_until_succeeds(
 209         "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
 210         + "jq '.data.result[0].value[1]' | grep '\"4\"'"
 211       )
 212   '';
 213 })