nixos/tests/hadoop/hdfs.nix

   1 # Test a minimal HDFS cluster with no HA
   2 import ../make-test-python.nix (
   3   { package, lib, ... }:
   4   {
   5     name = "hadoop-hdfs";
   6
   7     nodes =
   8       let
   9         coreSite = {
  10           "fs.defaultFS" = "hdfs://namenode:8020";
  11           "hadoop.proxyuser.httpfs.groups" = "*";
  12           "hadoop.proxyuser.httpfs.hosts" = "*";
  13         };
  14       in
  15       {
  16         namenode =
  17           { pkgs, ... }:
  18           {
  19             services.hadoop = {
  20               inherit package;
  21               hdfs = {
  22                 namenode = {
  23                   enable = true;
  24                   openFirewall = true;
  25                   formatOnInit = true;
  26                 };
  27                 httpfs = {
  28                   # The NixOS hadoop module only support webHDFS on 3.3 and newer
  29                   enable = lib.mkIf (lib.versionAtLeast package.version "3.3") true;
  30                   openFirewall = true;
  31                 };
  32               };
  33               inherit coreSite;
  34             };
  35           };
  36         datanode =
  37           { pkgs, ... }:
  38           {
  39             virtualisation.diskSize = 4096;
  40             services.hadoop = {
  41               inherit package;
  42               hdfs.datanode = {
  43                 enable = true;
  44                 openFirewall = true;
  45                 dataDirs = [
  46                   {
  47                     type = "DISK";
  48                     path = "/tmp/dn1";
  49                   }
  50                 ];
  51               };
  52               inherit coreSite;
  53             };
  54           };
  55       };
  56
  57     testScript =
  58       ''
  59         start_all()
  60
  61         namenode.wait_for_unit("hdfs-namenode")
  62         namenode.wait_for_unit("network.target")
  63         namenode.wait_for_open_port(8020)
  64         namenode.succeed("systemd-cat ss -tulpne")
  65         namenode.succeed("systemd-cat cat /etc/hadoop*/hdfs-site.xml")
  66         namenode.wait_for_open_port(9870)
  67
  68         datanode.wait_for_unit("hdfs-datanode")
  69         datanode.wait_for_unit("network.target")
  70       ''
  71       + (
  72         if lib.versionAtLeast package.version "3" then
  73           ''
  74             datanode.wait_for_open_port(9864)
  75             datanode.wait_for_open_port(9866)
  76             datanode.wait_for_open_port(9867)
  77
  78             datanode.succeed("curl -f http://datanode:9864")
  79           ''
  80         else
  81           ''
  82             datanode.wait_for_open_port(50075)
  83             datanode.wait_for_open_port(50010)
  84             datanode.wait_for_open_port(50020)
  85
  86             datanode.succeed("curl -f http://datanode:50075")
  87           ''
  88       )
  89       + ''
  90         namenode.succeed("curl -f http://namenode:9870")
  91
  92         datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
  93         datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
  94         assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
  95
  96       ''
  97       + lib.optionalString (lib.versionAtLeast package.version "3.3") ''
  98         namenode.wait_for_unit("hdfs-httpfs")
  99         namenode.wait_for_open_port(14000)
 100         assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
 101       '';
 102   }
 103 )