nixos/tests/prometheus/federation.nix

   1 import ../make-test-python.nix (
   2   { lib, pkgs, ... }:
   3
   4   {
   5     name = "prometheus-federation";
   6
   7     nodes = {
   8       global1 =
   9         { config, pkgs, ... }:
  10         {
  11           environment.systemPackages = [ pkgs.jq ];
  12
  13           networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
  14
  15           services.prometheus = {
  16             enable = true;
  17             globalConfig.scrape_interval = "2s";
  18
  19             scrapeConfigs = [
  20               {
  21                 job_name = "federate";
  22                 honor_labels = true;
  23                 metrics_path = "/federate";
  24
  25                 params = {
  26                   "match[]" = [
  27                     "{job=\"node\"}"
  28                     "{job=\"prometheus\"}"
  29                   ];
  30                 };
  31
  32                 static_configs = [
  33                   {
  34                     targets = [
  35                       "prometheus1:${toString config.services.prometheus.port}"
  36                       "prometheus2:${toString config.services.prometheus.port}"
  37                     ];
  38                   }
  39                 ];
  40               }
  41               {
  42                 job_name = "prometheus";
  43                 static_configs = [
  44                   {
  45                     targets = [
  46                       "global1:${toString config.services.prometheus.port}"
  47                       "global2:${toString config.services.prometheus.port}"
  48                     ];
  49                   }
  50                 ];
  51               }
  52             ];
  53           };
  54         };
  55
  56       global2 =
  57         { config, pkgs, ... }:
  58         {
  59           environment.systemPackages = [ pkgs.jq ];
  60
  61           networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
  62
  63           services.prometheus = {
  64             enable = true;
  65             globalConfig.scrape_interval = "2s";
  66
  67             scrapeConfigs = [
  68               {
  69                 job_name = "federate";
  70                 honor_labels = true;
  71                 metrics_path = "/federate";
  72
  73                 params = {
  74                   "match[]" = [
  75                     "{job=\"node\"}"
  76                     "{job=\"prometheus\"}"
  77                   ];
  78                 };
  79
  80                 static_configs = [
  81                   {
  82                     targets = [
  83                       "prometheus1:${toString config.services.prometheus.port}"
  84                       "prometheus2:${toString config.services.prometheus.port}"
  85                     ];
  86                   }
  87                 ];
  88               }
  89               {
  90                 job_name = "prometheus";
  91                 static_configs = [
  92                   {
  93                     targets = [
  94                       "global1:${toString config.services.prometheus.port}"
  95                       "global2:${toString config.services.prometheus.port}"
  96                     ];
  97                   }
  98                 ];
  99               }
 100             ];
 101           };
 102         };
 103
 104       prometheus1 =
 105         { config, pkgs, ... }:
 106         {
 107           environment.systemPackages = [ pkgs.jq ];
 108
 109           networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 110
 111           services.prometheus = {
 112             enable = true;
 113             globalConfig.scrape_interval = "2s";
 114
 115             scrapeConfigs = [
 116               {
 117                 job_name = "node";
 118                 static_configs = [
 119                   {
 120                     targets = [
 121                       "node1:${toString config.services.prometheus.exporters.node.port}"
 122                     ];
 123                   }
 124                 ];
 125               }
 126               {
 127                 job_name = "prometheus";
 128                 static_configs = [
 129                   {
 130                     targets = [
 131                       "prometheus1:${toString config.services.prometheus.port}"
 132                     ];
 133                   }
 134                 ];
 135               }
 136             ];
 137           };
 138         };
 139
 140       prometheus2 =
 141         { config, pkgs, ... }:
 142         {
 143           environment.systemPackages = [ pkgs.jq ];
 144
 145           networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
 146
 147           services.prometheus = {
 148             enable = true;
 149             globalConfig.scrape_interval = "2s";
 150
 151             scrapeConfigs = [
 152               {
 153                 job_name = "node";
 154                 static_configs = [
 155                   {
 156                     targets = [
 157                       "node2:${toString config.services.prometheus.exporters.node.port}"
 158                     ];
 159                   }
 160                 ];
 161               }
 162               {
 163                 job_name = "prometheus";
 164                 static_configs = [
 165                   {
 166                     targets = [
 167                       "prometheus2:${toString config.services.prometheus.port}"
 168                     ];
 169                   }
 170                 ];
 171               }
 172             ];
 173           };
 174         };
 175
 176       node1 =
 177         { config, pkgs, ... }:
 178         {
 179           services.prometheus.exporters.node = {
 180             enable = true;
 181             openFirewall = true;
 182           };
 183         };
 184
 185       node2 =
 186         { config, pkgs, ... }:
 187         {
 188           services.prometheus.exporters.node = {
 189             enable = true;
 190             openFirewall = true;
 191           };
 192         };
 193     };
 194
 195     testScript = ''
 196       for machine in node1, node2:
 197         machine.wait_for_unit("prometheus-node-exporter")
 198         machine.wait_for_open_port(9100)
 199
 200       for machine in prometheus1, prometheus2, global1, global2:
 201         machine.wait_for_unit("prometheus")
 202         machine.wait_for_open_port(9090)
 203
 204       # Verify both servers got the same data from the exporter
 205       for machine in prometheus1, prometheus2:
 206         machine.wait_until_succeeds(
 207           "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
 208           + "jq '.data.result[0].value[1]' | grep '\"1\"'"
 209         )
 210         machine.wait_until_succeeds(
 211           "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
 212           + "jq '.data.result[0].value[1]' | grep '\"1\"'"
 213         )
 214
 215       for machine in global1, global2:
 216         machine.wait_until_succeeds(
 217           "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\})' | "
 218           + "jq '.data.result[0].value[1]' | grep '\"2\"'"
 219         )
 220
 221         machine.wait_until_succeeds(
 222           "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(prometheus_build_info)' | "
 223           + "jq '.data.result[0].value[1]' | grep '\"4\"'"
 224         )
 225     '';
 226   }
 227 )