pkgs/development/tools/parsing/tree-sitter/update.nix

   1 { writeShellScript
   2 , writeText
   3 , writers
   4 , nix-prefetch-git
   5 , formats
   6 , lib
   7 , coreutils
   8 , curl
   9 , xe
  10 }:
  11
  12 # Grammar list:
  13 # https://github.com/tree-sitter/tree-sitter/blob/master/docs/index.md
  14
  15 let
  16   # Grammars we want to fetch from the tree-sitter github orga
  17   knownTreeSitterOrgGrammarRepos = [
  18     "tree-sitter-javascript"
  19     "tree-sitter-c"
  20     "tree-sitter-json"
  21     "tree-sitter-cpp"
  22     "tree-sitter-ruby"
  23     "tree-sitter-go"
  24     "tree-sitter-c-sharp"
  25     "tree-sitter-python"
  26     "tree-sitter-typescript"
  27     "tree-sitter-rust"
  28     "tree-sitter-bash"
  29     "tree-sitter-php"
  30     "tree-sitter-java"
  31     "tree-sitter-scala"
  32     "tree-sitter-ocaml"
  33     "tree-sitter-julia"
  34     "tree-sitter-html"
  35     "tree-sitter-haskell"
  36     "tree-sitter-regex"
  37     "tree-sitter-css"
  38     "tree-sitter-verilog"
  39     "tree-sitter-jsdoc"
  40     "tree-sitter-ql"
  41     "tree-sitter-ql-dbscheme"
  42     "tree-sitter-embedded-template"
  43     "tree-sitter-tsq"
  44     "tree-sitter-toml"
  45   ];
  46   knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
  47
  48   # repos of the tree-sitter github orga we want to ignore (not grammars)
  49   ignoredTreeSitterOrgRepos = [
  50     "tree-sitter"
  51     "tree-sitter-cli"
  52     # this is the haskell language bindings, tree-sitter-haskell is the grammar
  53     "haskell-tree-sitter"
  54     # this is the ruby language bindings, tree-sitter-ruby is the grammar
  55     "ruby-tree-sitter.old"
  56     # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
  57     "rust-tree-sitter"
  58     # this is the nodejs language bindings, tree-sitter-javascript is the grammar
  59     "node-tree-sitter"
  60     # this is the python language bindings, tree-sitter-python is the grammar
  61     "py-tree-sitter"
  62     # afl fuzzing for tree sitter
  63     "afl-tree-sitter"
  64     # this is the kotlin language bindings, tree-sitter-kotlin is the grammar
  65     "kotlin-tree-sitter"
  66     # this is the go language bindings, tree-sitter-go is the grammar
  67     "go-tree-sitter"
  68     # this is the java language bindings, tree-sitter-java is the grammar
  69     "java-tree-sitter"
  70     # archived
  71     "highlight-schema"
  72     # website
  73     "tree-sitter.github.io"
  74     # not maintained
  75     "tree-sitter-razor"
  76     # rust library for constructing arbitrary graph structures from source code
  77     "tree-sitter-graph"
  78     # abandoned
  79     "tree-sitter-swift"
  80     # abandoned
  81     "tree-sitter-agda"
  82     # abandoned
  83     "tree-sitter-fluent"
  84     # to unblock my update
  85     "csharp-tree-sitter"
  86     # (experimental) java bindings to the Tree-sitter parsing library
  87     "java-tree-sitter"
  88     # go bindings to the Tree-sitter parsing library
  89     "go-tree-sitter"
  90     # kotlin bindings to the Tree-sitter parsing library
  91     "kotlin-tree-sitter"
  92     # not ready to be used
  93     "zig-tree-sitter"
  94
  95
  96     # Non-grammar repositories
  97     ".github"
  98     "fuzz-action"
  99     "parse-action"
 100     "parser-setup-action"
 101     "parser-test-action"
 102     "parser-update-action"
 103     "setup-action"
 104     "workflows"
 105   ];
 106   ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
 107
 108   # Additional grammars that are not in the official github orga.
 109   # If you need a grammar that already exists in the official orga,
 110   # make sure to give it a different name.
 111   otherGrammars = {
 112     "tree-sitter-bitbake" = {
 113       orga = "amaanq";
 114       repo = "tree-sitter-bitbake";
 115     };
 116     "tree-sitter-beancount" = {
 117       orga = "polarmutex";
 118       repo = "tree-sitter-beancount";
 119     };
 120     "tree-sitter-bqn" = {
 121       orga = "shnarazk";
 122       repo = "tree-sitter-bqn";
 123     };
 124     "tree-sitter-clojure" = {
 125       orga = "sogaiu";
 126       repo = "tree-sitter-clojure";
 127     };
 128     "tree-sitter-comment" = {
 129       orga = "stsewd";
 130       repo = "tree-sitter-comment";
 131     };
 132     "tree-sitter-dart" = {
 133       orga = "usernobody14";
 134       repo = "tree-sitter-dart";
 135     };
 136     "tree-sitter-elisp" = {
 137       orga = "wilfred";
 138       repo = "tree-sitter-elisp";
 139     };
 140     "tree-sitter-just" = {
 141       orga = "IndianBoy42";
 142       repo = "tree-sitter-just";
 143     };
 144     "tree-sitter-nix" = {
 145       orga = "cstrahan";
 146       repo = "tree-sitter-nix";
 147     };
 148     "tree-sitter-latex" = {
 149       orga = "latex-lsp";
 150       repo = "tree-sitter-latex";
 151     };
 152     "tree-sitter-lua" = {
 153       orga = "MunifTanjim";
 154       repo = "tree-sitter-lua";
 155     };
 156     "tree-sitter-fennel" = {
 157       orga = "travonted";
 158       repo = "tree-sitter-fennel";
 159     };
 160     "tree-sitter-make" = {
 161       orga = "alemuller";
 162       repo = "tree-sitter-make";
 163     };
 164     "tree-sitter-markdown" = {
 165       orga = "MDeiml";
 166       repo = "tree-sitter-markdown";
 167     };
 168     "tree-sitter-proto" = {
 169       orga = "mitchellh";
 170       repo = "tree-sitter-proto";
 171     };
 172     "tree-sitter-rego" = {
 173       orga = "FallenAngel97";
 174       repo = "tree-sitter-rego";
 175     };
 176     "tree-sitter-rst" = {
 177       orga = "stsewd";
 178       repo = "tree-sitter-rst";
 179     };
 180     "tree-sitter-svelte" = {
 181       orga = "Himujjal";
 182       repo = "tree-sitter-svelte";
 183     };
 184     "tree-sitter-sql" = {
 185       orga = "derekstride";
 186       repo = "tree-sitter-sql";
 187       branch = "gh-pages";
 188     };
 189     "tree-sitter-talon" = {
 190       orga = "wenkokke";
 191       repo = "tree-sitter-talon";
 192     };
 193     "tree-sitter-typst" = {
 194       orga = "uben0";
 195       repo = "tree-sitter-typst";
 196     };
 197     "tree-sitter-vim" = {
 198       orga = "vigoux";
 199       repo = "tree-sitter-viml";
 200     };
 201     "tree-sitter-yaml" = {
 202       orga = "ikatyang";
 203       repo = "tree-sitter-yaml";
 204     };
 205     "tree-sitter-zig" = {
 206       orga = "maxxnino";
 207       repo = "tree-sitter-zig";
 208     };
 209     "tree-sitter-fish" = {
 210       orga = "ram02z";
 211       repo = "tree-sitter-fish";
 212     };
 213     "tree-sitter-dot" = {
 214       orga = "rydesun";
 215       repo = "tree-sitter-dot";
 216     };
 217     "tree-sitter-norg" = {
 218       orga = "nvim-neorg";
 219       repo = "tree-sitter-norg";
 220     };
 221     "tree-sitter-norg-meta" = {
 222       orga = "nvim-neorg";
 223       repo = "tree-sitter-norg-meta";
 224     };
 225     "tree-sitter-commonlisp" = {
 226       orga = "thehamsta";
 227       repo = "tree-sitter-commonlisp";
 228     };
 229     "tree-sitter-cuda" = {
 230       orga = "thehamsta";
 231       repo = "tree-sitter-cuda";
 232     };
 233     "tree-sitter-glsl" = {
 234       orga = "thehamsta";
 235       repo = "tree-sitter-glsl";
 236     };
 237     "tree-sitter-dockerfile" = {
 238       orga = "camdencheek";
 239       repo = "tree-sitter-dockerfile";
 240     };
 241     "tree-sitter-ledger" = {
 242       orga = "cbarrete";
 243       repo = "tree-sitter-ledger";
 244     };
 245     "tree-sitter-gomod" = {
 246       orga = "camdencheek";
 247       repo = "tree-sitter-go-mod";
 248     };
 249     "tree-sitter-gowork" = {
 250       orga = "omertuc";
 251       repo = "tree-sitter-go-work";
 252     };
 253     "tree-sitter-graphql" = {
 254       orga = "bkegley";
 255       repo = "tree-sitter-graphql";
 256     };
 257     "tree-sitter-pgn" = {
 258       orga = "rolandwalker";
 259       repo = "tree-sitter-pgn";
 260     };
 261     "tree-sitter-perl" = {
 262       orga = "ganezdragon";
 263       repo = "tree-sitter-perl";
 264     };
 265     "tree-sitter-kotlin" = {
 266       orga = "fwcd";
 267       repo = "tree-sitter-kotlin";
 268     };
 269     "tree-sitter-scss" = {
 270       orga = "serenadeai";
 271       repo = "tree-sitter-scss";
 272     };
 273     "tree-sitter-erlang" = {
 274       orga = "abstractmachineslab";
 275       repo = "tree-sitter-erlang";
 276     };
 277     "tree-sitter-elixir" = {
 278       orga = "elixir-lang";
 279       repo = "tree-sitter-elixir";
 280     };
 281     "tree-sitter-surface" = {
 282       orga = "connorlay";
 283       repo = "tree-sitter-surface";
 284     };
 285     "tree-sitter-eex" = {
 286       orga = "connorlay";
 287       repo = "tree-sitter-eex";
 288     };
 289     "tree-sitter-heex" = {
 290       orga = "connorlay";
 291       repo = "tree-sitter-heex";
 292     };
 293     "tree-sitter-supercollider" = {
 294       orga = "madskjeldgaard";
 295       repo = "tree-sitter-supercollider";
 296     };
 297     "tree-sitter-tlaplus" = {
 298       orga = "tlaplus-community";
 299       repo = "tree-sitter-tlaplus";
 300     };
 301     "tree-sitter-glimmer" = {
 302       orga = "alexlafroscia";
 303       repo = "tree-sitter-glimmer";
 304     };
 305     "tree-sitter-pug" = {
 306       orga = "zealot128";
 307       repo = "tree-sitter-pug";
 308     };
 309     "tree-sitter-vue" = {
 310       orga = "ikatyang";
 311       repo = "tree-sitter-vue";
 312     };
 313     "tree-sitter-elm" = {
 314       orga = "elm-tooling";
 315       repo = "tree-sitter-elm";
 316     };
 317     "tree-sitter-yang" = {
 318       orga = "hubro";
 319       repo = "tree-sitter-yang";
 320     };
 321     "tree-sitter-query" = {
 322       orga = "nvim-treesitter";
 323       repo = "tree-sitter-query";
 324     };
 325     "tree-sitter-sparql" = {
 326       orga = "bonabeavis";
 327       repo = "tree-sitter-sparql";
 328     };
 329     "tree-sitter-gdscript" = {
 330       orga = "prestonknopp";
 331       repo = "tree-sitter-gdscript";
 332     };
 333     "tree-sitter-godot-resource" = {
 334       orga = "prestonknopp";
 335       repo = "tree-sitter-godot-resource";
 336     };
 337     "tree-sitter-turtle" = {
 338       orga = "bonabeavis";
 339       repo = "tree-sitter-turtle";
 340     };
 341     "tree-sitter-devicetree" = {
 342       orga = "joelspadin";
 343       repo = "tree-sitter-devicetree";
 344     };
 345     "tree-sitter-r" = {
 346       orga = "r-lib";
 347       repo = "tree-sitter-r";
 348     };
 349     "tree-sitter-bibtex" = {
 350       orga = "latex-lsp";
 351       repo = "tree-sitter-bibtex";
 352     };
 353     "tree-sitter-fortran" = {
 354       orga = "stadelmanma";
 355       repo = "tree-sitter-fortran";
 356     };
 357     "tree-sitter-cmake" = {
 358       orga = "uyha";
 359       repo = "tree-sitter-cmake";
 360     };
 361     "tree-sitter-janet-simple" = {
 362       orga = "sogaiu";
 363       repo = "tree-sitter-janet-simple";
 364     };
 365     "tree-sitter-json5" = {
 366       orga = "joakker";
 367       repo = "tree-sitter-json5";
 368     };
 369     "tree-sitter-pioasm" = {
 370       orga = "leo60228";
 371       repo = "tree-sitter-pioasm";
 372     };
 373     "tree-sitter-hjson" = {
 374       orga = "winston0410";
 375       repo = "tree-sitter-hjson";
 376     };
 377     "tree-sitter-llvm" = {
 378       orga = "benwilliamgraham";
 379       repo = "tree-sitter-llvm";
 380     };
 381     "tree-sitter-http" = {
 382       orga = "ntbbloodbath";
 383       repo = "tree-sitter-http";
 384     };
 385     "tree-sitter-prisma" = {
 386       orga = "victorhqc";
 387       repo = "tree-sitter-prisma";
 388     };
 389     "tree-sitter-org-nvim" = {
 390       orga = "milisims";
 391       repo = "tree-sitter-org";
 392     };
 393     "tree-sitter-hcl" = {
 394       orga = "MichaHoffmann";
 395       repo = "tree-sitter-hcl";
 396     };
 397     "tree-sitter-scheme" = {
 398       orga = "6cdh";
 399       repo = "tree-sitter-scheme";
 400     };
 401     "tree-sitter-tiger" = {
 402       orga = "ambroisie";
 403       repo = "tree-sitter-tiger";
 404     };
 405     "tree-sitter-nickel" = {
 406       orga = "nickel-lang";
 407       repo = "tree-sitter-nickel";
 408     };
 409     "tree-sitter-smithy" = {
 410       orga = "indoorvivants";
 411       repo = "tree-sitter-smithy";
 412     };
 413     "tree-sitter-jsonnet" = {
 414       orga = "sourcegraph";
 415       repo = "tree-sitter-jsonnet";
 416     };
 417     "tree-sitter-solidity" = {
 418       orga = "JoranHonig";
 419       repo = "tree-sitter-solidity";
 420     };
 421     "tree-sitter-nu" = {
 422       orga = "nushell";
 423       repo = "tree-sitter-nu";
 424     };
 425     "tree-sitter-cue" = {
 426       orga = "eonpatapon";
 427       repo = "tree-sitter-cue";
 428     };
 429     "tree-sitter-uiua" = {
 430       orga = "shnarazk";
 431       repo = "tree-sitter-uiua";
 432     };
 433     "tree-sitter-wing" = {
 434       orga = "winglang";
 435       repo = "tree-sitter-wing";
 436     };
 437     "tree-sitter-wgsl" = {
 438       orga = "szebniok";
 439       repo = "tree-sitter-wgsl";
 440     };
 441     "tree-sitter-templ" = {
 442       orga = "vrischmann";
 443       repo = "tree-sitter-templ";
 444     };
 445     "tree-sitter-gleam" = {
 446       orga = "gleam-lang";
 447       repo = "tree-sitter-gleam";
 448     };
 449     "tree-sitter-koka" = {
 450       orga = "mtoohey31";
 451       repo = "tree-sitter-koka";
 452     };
 453     "tree-sitter-earthfile" = {
 454       orga = "glehmann";
 455       repo = "tree-sitter-earthfile";
 456     };
 457     "tree-sitter-river" = {
 458       orga = "grafana";
 459       repo = "tree-sitter-river";
 460     };
 461     "tree-sitter-twig" = {
 462       orga = "kaermorchen";
 463       repo = "tree-sitter-twig";
 464     };
 465     "tree-sitter-hyprlang" = {
 466       orga = "tree-sitter-grammars";
 467       repo = "tree-sitter-hyprlang";
 468     };
 469     "tree-sitter-kdl" = {
 470       orga = "tree-sitter-grammars";
 471       repo = "tree-sitter-kdl";
 472     };
 473     "tree-sitter-tera" = {
 474       orga = "uncenter";
 475       repo = "tree-sitter-tera";
 476     };
 477   };
 478
 479   allGrammars =
 480     let
 481       treeSitterOrgaGrammars =
 482         lib.listToAttrs (map
 483           (repo:
 484             {
 485               name = repo;
 486               value = {
 487                 orga = "tree-sitter";
 488                 inherit repo;
 489               };
 490             })
 491           knownTreeSitterOrgGrammarRepos);
 492
 493     in
 494     lib.attrsets.unionOfDisjoint otherGrammars treeSitterOrgaGrammars;
 495
 496
 497
 498   jsonFile = name: val: (formats.json { }).generate name val;
 499
 500   # implementation of the updater
 501   updateImpl = passArgs "updateImpl-with-args"
 502     {
 503       binaries = {
 504         curl = "${curl}/bin/curl";
 505         nix-prefetch-git = "${nix-prefetch-git}/bin/nix-prefetch-git";
 506         printf = "${coreutils}/bin/printf";
 507       };
 508       inherit
 509         knownTreeSitterOrgGrammarRepos
 510         ignoredTreeSitterOrgRepos
 511         ;
 512     }
 513     (writers.writePython3 "updateImpl"
 514       {
 515         flakeIgnore = [ "E501" ];
 516       } ./update_impl.py);
 517
 518   # Pass the given arguments to the command, in the ARGS environment variable.
 519   # The arguments are just a json object that should be available in the script.
 520   passArgs = name: argAttrs: script: writeShellScript name ''
 521     env ARGS="$(< ${jsonFile "${name}-args" argAttrs})" \
 522       ${script} "$@"
 523   '';
 524
 525   foreachSh = attrs: f:
 526     lib.concatMapStringsSep "\n" f
 527       (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
 528
 529   jsonNewlines = lib.concatMapStringsSep "\n" (lib.generators.toJSON { });
 530
 531   # Run the given script for each of the attr list.
 532   # The attrs are passed to the script as a json value.
 533   forEachParallel = name: script: listOfAttrs: writeShellScript "for-each-parallel.sh" ''
 534     < ${writeText "${name}.json" (jsonNewlines listOfAttrs)} \
 535       ${xe}/bin/xe -F -j5 ${script} {}
 536   '';
 537
 538   # The output directory in the current source tree.
 539   # This will depend on your local environment, but that is intentional.
 540   outputDir = "${toString ./.}/grammars";
 541
 542   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
 543      set -euo pipefail
 544     ${updateImpl} fetch-and-check-tree-sitter-repos '{}'
 545      echo "writing files to ${outputDir}" 1>&2
 546      mkdir -p "${outputDir}"
 547      ${forEachParallel
 548          "repos-to-fetch"
 549          (writeShellScript "fetch-repo" ''
 550              ${updateImpl} fetch-repo "$1"
 551          '')
 552          (lib.mapAttrsToList
 553            (nixRepoAttrName: attrs: attrs // {
 554              inherit
 555                nixRepoAttrName
 556                outputDir;
 557            })
 558            allGrammars)
 559      }
 560      ${updateImpl} print-all-grammars-nix-file "$(< ${
 561          jsonFile "all-grammars.json" {
 562            allGrammars =
 563              (lib.mapAttrsToList
 564                (nixRepoAttrName: attrs: attrs // {
 565                  inherit nixRepoAttrName;
 566                })
 567                allGrammars);
 568            inherit outputDir;
 569          }
 570      })"
 571   '';
 572
 573
 574 in
 575 update-all-grammars