maintainers/scripts/copy-tarballs.pl

   1 #! /usr/bin/env nix-shell
   2 #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nixUnstable nixUnstable.perl-bindings
   3
   4 # This command uploads tarballs to tarballs.nixos.org, the
   5 # content-addressed cache used by fetchurl as a fallback for when
   6 # upstream tarballs disappear or change. Usage:
   7 #
   8 # 1) To upload one or more files:
   9 #
  10 #    $ copy-tarballs.pl --file /path/to/tarball.tar.gz
  11 #
  12 # 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
  13 #
  14 #    $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
  15
  16 use strict;
  17 use warnings;
  18 use File::Basename;
  19 use File::Path;
  20 use File::Slurp;
  21 use JSON;
  22 use Net::Amazon::S3;
  23 use Nix::Store;
  24
  25 isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation
  26
  27 sub usage {
  28     die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
  29 }
  30
  31 my $dryRun = 0;
  32 my $expr;
  33 my @fileNames;
  34 my $exclude;
  35
  36 while (@ARGV) {
  37     my $flag = shift @ARGV;
  38
  39     if ($flag eq "--expr") {
  40         $expr = shift @ARGV or die "--expr requires an argument";
  41     } elsif ($flag eq "--file") {
  42         @fileNames = @ARGV;
  43         last;
  44     } elsif ($flag eq "--dry-run") {
  45         $dryRun = 1;
  46     } elsif ($flag eq "--exclude") {
  47         $exclude = shift @ARGV or die "--exclude requires an argument";
  48     } else {
  49         usage();
  50     }
  51 }
  52
  53
  54 # S3 setup.
  55 my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "AWS_ACCESS_KEY_ID not set\n";
  56 my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "AWS_SECRET_ACCESS_KEY not set\n";
  57
  58 my $s3 = Net::Amazon::S3->new(
  59     { aws_access_key_id     => $aws_access_key_id,
  60       aws_secret_access_key => $aws_secret_access_key,
  61       retry                 => 1,
  62       host                  => "s3-eu-west-1.amazonaws.com",
  63     });
  64
  65 my $bucket = $s3->bucket("nixpkgs-tarballs") or die;
  66
  67 my $doWrite = 0;
  68 my $cacheFile = ($ENV{"HOME"} or die "\$HOME is not set") . "/.cache/nix/copy-tarballs";
  69 my %cache;
  70 $cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
  71 $doWrite = 1;
  72
  73 END() {
  74     File::Path::mkpath(dirname($cacheFile), 0, 0755);
  75     write_file($cacheFile, map { "$_\n" } keys %cache) if $doWrite;
  76 }
  77
  78 sub alreadyMirrored {
  79     my ($algo, $hash) = @_;
  80     my $key = "$algo/$hash";
  81     return 1 if defined $cache{$key};
  82     my $res = defined $bucket->get_key($key);
  83     $cache{$key} = 1 if $res;
  84     return $res;
  85 }
  86
  87 sub uploadFile {
  88     my ($fn, $name) = @_;
  89
  90     my $md5_16 = hashFile("md5", 0, $fn) or die;
  91     my $sha1_16 = hashFile("sha1", 0, $fn) or die;
  92     my $sha256_32 = hashFile("sha256", 1, $fn) or die;
  93     my $sha256_16 = hashFile("sha256", 0, $fn) or die;
  94     my $sha512_32 = hashFile("sha512", 1, $fn) or die;
  95     my $sha512_16 = hashFile("sha512", 0, $fn) or die;
  96
  97     my $mainKey = "sha512/$sha512_16";
  98
  99     # Create redirects from the other hash types.
 100     sub redirect {
 101         my ($name, $dest) = @_;
 102         #print STDERR "linking $name to $dest...\n";
 103         $bucket->add_key($name, "", {
 104             'x-amz-website-redirect-location' => "/" . $dest,
 105             'x-amz-acl' => "public-read"
 106         })
 107             or die "failed to create redirect from $name to $dest\n";
 108         $cache{$name} = 1;
 109     }
 110     redirect "md5/$md5_16", $mainKey;
 111     redirect "sha1/$sha1_16", $mainKey;
 112     redirect "sha256/$sha256_32", $mainKey;
 113     redirect "sha256/$sha256_16", $mainKey;
 114     redirect "sha512/$sha512_32", $mainKey;
 115
 116     # Upload the file as sha512/<hash-in-base-16>.
 117     print STDERR "uploading $fn to $mainKey...\n";
 118     $bucket->add_key_filename($mainKey, $fn, {
 119         'x-amz-meta-original-name' => $name,
 120         'x-amz-acl' => "public-read"
 121     })
 122         or die "failed to upload $fn to $mainKey\n";
 123     $cache{$mainKey} = 1;
 124 }
 125
 126 if (scalar @fileNames) {
 127     my $res = 0;
 128     foreach my $fn (@fileNames) {
 129         eval {
 130             if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
 131                 print STDERR "$fn is already mirrored\n";
 132             } else {
 133                 uploadFile($fn, basename $fn);
 134             }
 135         };
 136         if ($@) {
 137             warn "$@";
 138             $res = 1;
 139         }
 140     }
 141     exit $res;
 142 }
 143
 144 elsif (defined $expr) {
 145
 146     # Evaluate find-tarballs.nix.
 147     my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
 148                    "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
 149                    "--arg", "expr", $expr);
 150     my $stdout = <JSON>;
 151     waitpid($pid, 0);
 152     die "$0: evaluation failed\n" if $?;
 153     close JSON;
 154
 155     my $fetches = decode_json($stdout);
 156
 157     print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
 158
 159     # Check every fetchurl call discovered by find-tarballs.nix.
 160     my $mirrored = 0;
 161     my $have = 0;
 162     foreach my $fetch (sort { $a->{url} cmp $b->{url} } @{$fetches}) {
 163         my $url = $fetch->{url};
 164         my $algo = $fetch->{type};
 165         my $hash = $fetch->{hash};
 166         my $name = $fetch->{name};
 167         my $isPatch = $fetch->{isPatch};
 168
 169         if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
 170             $algo = $1;
 171             $hash = `nix hash to-base16 $hash` or die;
 172             chomp $hash;
 173         }
 174
 175         next unless $algo =~ /^[a-z0-9]+$/;
 176
 177         # Convert non-SRI base-64 to base-16.
 178         if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
 179             $hash = `nix hash to-base16 --type '$algo' $hash` or die;
 180             chomp $hash;
 181         }
 182
 183         if (defined $ENV{DEBUG}) {
 184             print "$url $algo $hash\n";
 185             next;
 186         }
 187
 188         if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
 189             print STDERR "skipping $url (unsupported scheme)\n";
 190             next;
 191         }
 192
 193         if ($isPatch) {
 194             print STDERR "skipping $url (support for patches is missing)\n";
 195             next;
 196         }
 197
 198         next if defined $exclude && $url =~ /$exclude/;
 199
 200         if (alreadyMirrored($algo, $hash)) {
 201             $have++;
 202             next;
 203         }
 204
 205         my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
 206
 207         print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
 208
 209         if ($dryRun) {
 210             $mirrored++;
 211             next;
 212         }
 213
 214         # Substitute the output.
 215         if (!isValidPath($storePath)) {
 216             system("nix-store", "-r", $storePath);
 217         }
 218
 219         # Otherwise download the file using nix-prefetch-url.
 220         if (!isValidPath($storePath)) {
 221             $ENV{QUIET} = 1;
 222             $ENV{PRINT_PATH} = 1;
 223             my $fh;
 224             my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
 225             waitpid($pid, 0) or die;
 226             if ($? != 0) {
 227                 print STDERR "failed to fetch $url: $?\n";
 228                 next;
 229             }
 230             <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
 231             if ($storePath ne $storePath2) {
 232                 warn "strange: $storePath != $storePath2\n";
 233                 next;
 234             }
 235         }
 236
 237         uploadFile($storePath, $url);
 238         $mirrored++;
 239     }
 240
 241     print STDERR "mirrored $mirrored files, already have $have files\n";
 242 }
 243
 244 else {
 245     usage();
 246 }