nixos/tests/quickwit.nix

   1 import ./make-test-python.nix ({ lib, pkgs, ... }:
   2
   3 let
   4   # Define an example Quickwit index schema,
   5   # and some `exampleDocs` below, to test if ingesting
   6   # and querying works as expected.
   7   index_yaml = ''
   8     version: 0.7
   9     index_id: example_server_logs
  10     doc_mapping:
  11       mode: dynamic
  12       field_mappings:
  13         - name: datetime
  14           type: datetime
  15           fast: true
  16           input_formats:
  17             - iso8601
  18           output_format: iso8601
  19           fast_precision: seconds
  20           fast: true
  21         - name: git
  22           type: text
  23           tokenizer: raw
  24         - name: hostname
  25           type: text
  26           tokenizer: raw
  27         - name: level
  28           type: text
  29           tokenizer: raw
  30         - name: message
  31           type: text
  32         - name: location
  33           type: text
  34         - name: source
  35           type: text
  36       timestamp_field: datetime
  37
  38     search_settings:
  39       default_search_fields: [message]
  40
  41     indexing_settings:
  42       commit_timeout_secs: 10
  43   '';
  44
  45   exampleDocs = ''
  46     {"datetime":"2024-05-03T02:36:41.017674444Z","git":"e6e1f087ce12065e44ed3b87b50784e6f9bcc2f9","hostname":"machine-1","level":"Info","message":"Processing request done","location":"path/to/server.c:6442:32","source":""}
  47     {"datetime":"2024-05-04T02:36:41.017674444Z","git":"e6e1f087ce12065e44ed3b87b50784e6f9bcc2f9","hostname":"machine-1","level":"Info","message":"Got exception processing request: HTTP 404","location":"path/to/server.c:6444:32","source":""}
  48     {"datetime":"2024-05-05T02:36:41.017674444Z","git":"e6e1f087ce12065e44ed3b87b50784e6f9bcc2f9","hostname":"machine-1","level":"Info","message":"Got exception processing request: HTTP 404","location":"path/to/server.c:6444:32","source":""}
  49     {"datetime":"2024-05-06T02:36:41.017674444Z","git":"e6e1f087ce12065e44ed3b87b50784e6f9bcc2f9","hostname":"machine-2","level":"Info","message":"Got exception processing request: HTTP 404","location":"path/to/server.c:6444:32","source":""}
  50   '';
  51 in
  52 {
  53   name = "quickwit";
  54   meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
  55
  56   nodes = {
  57     quickwit = { config, pkgs, ... }: {
  58       services.quickwit.enable = true;
  59     };
  60   };
  61
  62   testScript =
  63   ''
  64     quickwit.wait_for_unit("quickwit")
  65     quickwit.wait_for_open_port(7280)
  66     quickwit.wait_for_open_port(7281)
  67
  68     quickwit.wait_until_succeeds(
  69       "journalctl -o cat -u quickwit.service | grep 'version: ${pkgs.quickwit.version}'"
  70     )
  71
  72     quickwit.wait_until_succeeds(
  73       "journalctl -o cat -u quickwit.service | grep 'transitioned to ready state'"
  74     )
  75
  76     with subtest("verify UI installed"):
  77       machine.succeed("curl -sSf http://127.0.0.1:7280/ui/")
  78
  79     with subtest("injest and query data"):
  80       import json
  81
  82       # Test CLI ingestion
  83       print(machine.succeed('${pkgs.quickwit}/bin/quickwit index create --index-config ${pkgs.writeText "index.yaml" index_yaml}'))
  84       # Important to use `--wait`, otherwise the queries below race with index processing.
  85       print(machine.succeed('${pkgs.quickwit}/bin/quickwit index ingest --index example_server_logs --input-path ${pkgs.writeText "exampleDocs.json" exampleDocs} --wait'))
  86
  87       # Test CLI query
  88       cli_query_output = machine.succeed('${pkgs.quickwit}/bin/quickwit index search --index example_server_logs --query "exception"')
  89       print(cli_query_output)
  90
  91       # Assert query result is as expected.
  92       num_hits = len(json.loads(cli_query_output)["hits"])
  93       assert num_hits == 3, f"cli_query_output contains unexpected number of results: {num_hits}"
  94
  95       # Test API query
  96       api_query_output = machine.succeed('curl --fail http://127.0.0.1:7280/api/v1/example_server_logs/search?query=exception')
  97       print(api_query_output)
  98
  99     quickwit.log(quickwit.succeed(
 100       "systemd-analyze security quickwit.service | grep -v '✓'"
 101     ))
 102   '';
 103 })