nixos/tests/prometheus/config-reload.nix

   1 import ../make-test-python.nix ({ lib, pkgs, ... }:
   2
   3 {
   4   name = "prometheus-config-reload";
   5
   6   nodes = {
   7     prometheus = { config, pkgs, ... }: {
   8       environment.systemPackages = [ pkgs.jq ];
   9
  10       networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];
  11
  12       services.prometheus = {
  13         enable = true;
  14         enableReload = true;
  15         globalConfig.scrape_interval = "2s";
  16         scrapeConfigs = [
  17           {
  18             job_name = "prometheus";
  19             static_configs = [
  20               {
  21                 targets = [
  22                   "prometheus:${toString config.services.prometheus.port}"
  23                 ];
  24               }
  25             ];
  26           }
  27         ];
  28       };
  29
  30       specialisation = {
  31         "prometheus-config-change" = {
  32           configuration = {
  33             environment.systemPackages = [ pkgs.yq ];
  34
  35             # This configuration just adds a new prometheus job
  36             # to scrape the node_exporter metrics of the s3 machine.
  37             services.prometheus = {
  38               scrapeConfigs = [
  39                 {
  40                   job_name = "node";
  41                   static_configs = [
  42                     {
  43                       targets = [ "node:${toString config.services.prometheus.exporters.node.port}" ];
  44                     }
  45                   ];
  46                 }
  47               ];
  48             };
  49           };
  50         };
  51       };
  52     };
  53   };
  54
  55   testScript = ''
  56     prometheus.wait_for_unit("prometheus")
  57     prometheus.wait_for_open_port(9090)
  58
  59     # Check if switching to a NixOS configuration that changes the prometheus
  60     # configuration reloads (instead of restarts) prometheus before the switch
  61     # finishes successfully:
  62     with subtest("config change reloads prometheus"):
  63       import json
  64       # We check if prometheus has finished reloading by looking for the message
  65       # "Completed loading of configuration file" in the journal between the start
  66       # and finish of switching to the new NixOS configuration.
  67       #
  68       # To mark the start we record the journal cursor before starting the switch:
  69       cursor_before_switching = json.loads(
  70           prometheus.succeed("journalctl -n1 -o json --output-fields=__CURSOR")
  71       )["__CURSOR"]
  72
  73       # Now we switch:
  74       prometheus_config_change = prometheus.succeed(
  75           "readlink /run/current-system/specialisation/prometheus-config-change"
  76       ).strip()
  77       prometheus.succeed(prometheus_config_change + "/bin/switch-to-configuration test")
  78
  79       # Next we retrieve all logs since the start of switching:
  80       logs_after_starting_switching = prometheus.succeed(
  81           """
  82             journalctl --after-cursor='{cursor_before_switching}' -o json --output-fields=MESSAGE
  83           """.format(
  84               cursor_before_switching=cursor_before_switching
  85           )
  86       )
  87
  88       # Finally we check if the message "Completed loading of configuration file"
  89       # occurs before the "finished switching to system configuration" message:
  90       finished_switching_msg = (
  91           "finished switching to system configuration " + prometheus_config_change
  92       )
  93       reloaded_before_switching_finished = False
  94       finished_switching = False
  95       for log_line in logs_after_starting_switching.split("\n"):
  96           msg = json.loads(log_line)["MESSAGE"]
  97           if "Completed loading of configuration file" in msg:
  98               reloaded_before_switching_finished = True
  99           if msg == finished_switching_msg:
 100               finished_switching = True
 101               break
 102
 103       assert reloaded_before_switching_finished
 104       assert finished_switching
 105
 106       # Check if the reloaded config includes the new node job:
 107       prometheus.succeed(
 108         """
 109           curl -sf http://127.0.0.1:9090/api/v1/status/config \
 110             | jq -r .data.yaml \
 111             | yq '.scrape_configs | any(.job_name == "node")' \
 112             | grep true
 113         """
 114       )
 115   '';
 116 })