nixos/tests/kafka.nix

   1 { system ? builtins.currentSystem,
   2   config ? {},
   3   pkgs ? import ../.. { inherit system config; }
   4 }:
   5
   6 with pkgs.lib;
   7
   8 let
   9   makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({
  10     inherit name;
  11     meta = with pkgs.stdenv.lib.maintainers; {
  12       maintainers = [ nequissimus ];
  13     };
  14
  15     nodes = {
  16       zookeeper1 = { ... }: {
  17         services.zookeeper = {
  18           enable = true;
  19         };
  20
  21         networking.firewall.allowedTCPPorts = [ 2181 ];
  22         virtualisation.memorySize = 1024;
  23       };
  24       kafka = { ... }: {
  25         services.apache-kafka = {
  26           enable = true;
  27           extraProperties = ''
  28             offsets.topic.replication.factor = 1
  29             zookeeper.session.timeout.ms = 600000
  30           '';
  31           package = kafkaPackage;
  32           zookeeper = "zookeeper1:2181";
  33           # These are the default options, but UseCompressedOops doesn't work with 32bit JVM
  34           jvmOptions = [
  35             "-server" "-Xmx1G" "-Xms1G" "-XX:+UseParNewGC" "-XX:+UseConcMarkSweepGC" "-XX:+CMSClassUnloadingEnabled"
  36             "-XX:+CMSScavengeBeforeRemark" "-XX:+DisableExplicitGC" "-Djava.awt.headless=true" "-Djava.net.preferIPv4Stack=true"
  37           ] ++ optionals (! pkgs.stdenv.isi686 ) [ "-XX:+UseCompressedOops" ];
  38         };
  39
  40         networking.firewall.allowedTCPPorts = [ 9092 ];
  41         # i686 tests: qemu-system-i386 can simulate max 2047MB RAM (not 2048)
  42         virtualisation.memorySize = 2047;
  43       };
  44     };
  45
  46     testScript = ''
  47       start_all()
  48
  49       zookeeper1.wait_for_unit("default.target")
  50       zookeeper1.wait_for_unit("zookeeper.service")
  51       zookeeper1.wait_for_open_port(2181)
  52
  53       kafka.wait_for_unit("default.target")
  54       kafka.wait_for_unit("apache-kafka.service")
  55       kafka.wait_for_open_port(9092)
  56
  57       kafka.wait_until_succeeds(
  58           "${kafkaPackage}/bin/kafka-topics.sh --create "
  59           + "--zookeeper zookeeper1:2181 --partitions 1 "
  60           + "--replication-factor 1 --topic testtopic"
  61       )
  62       kafka.succeed(
  63           "echo 'test 1' | "
  64           + "${kafkaPackage}/bin/kafka-console-producer.sh "
  65           + "--broker-list localhost:9092 --topic testtopic"
  66       )
  67     '' + (if name == "kafka_0_9" then ''
  68       assert "test 1" in kafka.succeed(
  69           "${kafkaPackage}/bin/kafka-console-consumer.sh "
  70           + "--zookeeper zookeeper1:2181 --topic testtopic "
  71           + "--from-beginning --max-messages 1"
  72       )
  73     '' else ''
  74       assert "test 1" in kafka.succeed(
  75           "${kafkaPackage}/bin/kafka-console-consumer.sh "
  76           + "--bootstrap-server localhost:9092 --topic testtopic "
  77           + "--from-beginning --max-messages 1"
  78       )
  79     '');
  80   }) { inherit system; });
  81
  82 in with pkgs; {
  83   kafka_0_9  = makeKafkaTest "kafka_0_9"  apacheKafka_0_9;
  84   kafka_0_10 = makeKafkaTest "kafka_0_10" apacheKafka_0_10;
  85   kafka_0_11 = makeKafkaTest "kafka_0_11" apacheKafka_0_11;
  86   kafka_1_0  = makeKafkaTest "kafka_1_0"  apacheKafka_1_0;
  87   kafka_1_1  = makeKafkaTest "kafka_1_1"  apacheKafka_1_1;
  88   kafka_2_0  = makeKafkaTest "kafka_2_0"  apacheKafka_2_0;
  89   kafka_2_1  = makeKafkaTest "kafka_2_1"  apacheKafka_2_1;
  90   kafka_2_2  = makeKafkaTest "kafka_2_2"  apacheKafka_2_2;
  91   kafka_2_3  = makeKafkaTest "kafka_2_3"  apacheKafka_2_3;
  92   kafka_2_4  = makeKafkaTest "kafka_2_4"  apacheKafka_2_4;
  93 }