maintainers/scripts/copy-tarballs.pl

   1 #! /usr/bin/env nix-shell
   2 #! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nixUnstable nixUnstable.perl-bindings
   3
   4 # This command uploads tarballs to tarballs.nixos.org, the
   5 # content-addressed cache used by fetchurl as a fallback for when
   6 # upstream tarballs disappear or change. Usage:
   7 #
   8 # 1) To upload one or more files:
   9 #
  10 #    $ copy-tarballs.pl --file /path/to/tarball.tar.gz
  11 #
  12 # 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
  13 #
  14 #    $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
  15
  16 use strict;
  17 use warnings;
  18 use File::Basename;
  19 use File::Path;
  20 use File::Slurp;
  21 use JSON;
  22 use Net::Amazon::S3;
  23 use Nix::Store;
  24
  25 isValidPath("/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-foo"); # FIXME: forces Nix::Store initialisation
  26
  27 sub usage {
  28     die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
  29 }
  30
  31 my $dryRun = 0;
  32 my $expr;
  33 my @fileNames;
  34 my $exclude;
  35
  36 while (@ARGV) {
  37     my $flag = shift @ARGV;
  38
  39     if ($flag eq "--expr") {
  40         $expr = shift @ARGV or die "--expr requires an argument";
  41     } elsif ($flag eq "--file") {
  42         @fileNames = @ARGV;
  43         last;
  44     } elsif ($flag eq "--dry-run") {
  45         $dryRun = 1;
  46     } elsif ($flag eq "--exclude") {
  47         $exclude = shift @ARGV or die "--exclude requires an argument";
  48     } else {
  49         usage();
  50     }
  51 }
  52
  53 my $bucket;
  54
  55 if (not defined $ENV{DEBUG}) {
  56     # S3 setup.
  57     my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "AWS_ACCESS_KEY_ID not set\n";
  58     my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "AWS_SECRET_ACCESS_KEY not set\n";
  59
  60     my $s3 = Net::Amazon::S3->new(
  61         { aws_access_key_id     => $aws_access_key_id,
  62           aws_secret_access_key => $aws_secret_access_key,
  63           retry                 => 1,
  64           host                  => "s3-eu-west-1.amazonaws.com",
  65         });
  66
  67     $bucket = $s3->bucket("nixpkgs-tarballs") or die;
  68 }
  69
  70 my $doWrite = 0;
  71 my $cacheFile = ($ENV{"HOME"} or die "\$HOME is not set") . "/.cache/nix/copy-tarballs";
  72 my %cache;
  73 $cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
  74 $doWrite = 1;
  75
  76 END() {
  77     File::Path::mkpath(dirname($cacheFile), 0, 0755);
  78     write_file($cacheFile, map { "$_\n" } keys %cache) if $doWrite;
  79 }
  80
  81 sub alreadyMirrored {
  82     my ($algo, $hash) = @_;
  83     my $key = "$algo/$hash";
  84     return 1 if defined $cache{$key};
  85     my $res = defined $bucket->get_key($key);
  86     $cache{$key} = 1 if $res;
  87     return $res;
  88 }
  89
  90 sub uploadFile {
  91     my ($fn, $name) = @_;
  92
  93     my $md5_16 = hashFile("md5", 0, $fn) or die;
  94     my $sha1_16 = hashFile("sha1", 0, $fn) or die;
  95     my $sha256_32 = hashFile("sha256", 1, $fn) or die;
  96     my $sha256_16 = hashFile("sha256", 0, $fn) or die;
  97     my $sha512_32 = hashFile("sha512", 1, $fn) or die;
  98     my $sha512_16 = hashFile("sha512", 0, $fn) or die;
  99
 100     my $mainKey = "sha512/$sha512_16";
 101
 102     # Create redirects from the other hash types.
 103     sub redirect {
 104         my ($name, $dest) = @_;
 105         #print STDERR "linking $name to $dest...\n";
 106         $bucket->add_key($name, "", {
 107             'x-amz-website-redirect-location' => "/" . $dest,
 108             'x-amz-acl' => "public-read"
 109         })
 110             or die "failed to create redirect from $name to $dest\n";
 111         $cache{$name} = 1;
 112     }
 113     redirect "md5/$md5_16", $mainKey;
 114     redirect "sha1/$sha1_16", $mainKey;
 115     redirect "sha256/$sha256_32", $mainKey;
 116     redirect "sha256/$sha256_16", $mainKey;
 117     redirect "sha512/$sha512_32", $mainKey;
 118
 119     # Upload the file as sha512/<hash-in-base-16>.
 120     print STDERR "uploading $fn to $mainKey...\n";
 121     $bucket->add_key_filename($mainKey, $fn, {
 122         'x-amz-meta-original-name' => $name,
 123         'x-amz-acl' => "public-read"
 124     })
 125         or die "failed to upload $fn to $mainKey\n";
 126     $cache{$mainKey} = 1;
 127 }
 128
 129 if (scalar @fileNames) {
 130     my $res = 0;
 131     foreach my $fn (@fileNames) {
 132         eval {
 133             if (alreadyMirrored("sha512", hashFile("sha512", 0, $fn))) {
 134                 print STDERR "$fn is already mirrored\n";
 135             } else {
 136                 uploadFile($fn, basename $fn);
 137             }
 138         };
 139         if ($@) {
 140             warn "$@";
 141             $res = 1;
 142         }
 143     }
 144     exit $res;
 145 }
 146
 147 elsif (defined $expr) {
 148
 149     # Evaluate find-tarballs.nix.
 150     my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
 151                    "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
 152                    "--arg", "expr", $expr);
 153     my $stdout = <JSON>;
 154     waitpid($pid, 0);
 155     die "$0: evaluation failed\n" if $?;
 156     close JSON;
 157
 158     my $fetches = decode_json($stdout);
 159
 160     print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
 161
 162     # Check every fetchurl call discovered by find-tarballs.nix.
 163     my $mirrored = 0;
 164     my $have = 0;
 165     foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
 166         my $urls = $fetch->{urls};
 167         my $algo = $fetch->{type};
 168         my $hash = $fetch->{hash};
 169         my $name = $fetch->{name};
 170         my $isPatch = $fetch->{isPatch};
 171
 172         if ($isPatch) {
 173             print STDERR "skipping $urls->[0] (support for patches is missing)\n";
 174             next;
 175         }
 176
 177         if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
 178             $algo = $1;
 179             $hash = `nix hash to-base16 $hash` or die;
 180             chomp $hash;
 181         }
 182
 183         next unless $algo =~ /^[a-z0-9]+$/;
 184
 185         # Convert non-SRI base-64 to base-16.
 186         if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
 187             $hash = `nix hash to-base16 --type '$algo' $hash` or die;
 188             chomp $hash;
 189         }
 190
 191         my $storePath = makeFixedOutputPath(0, $algo, $hash, $name);
 192
 193         for my $url (@$urls) {
 194             if (defined $ENV{DEBUG}) {
 195                 print "$url $algo $hash\n";
 196                 next;
 197             }
 198
 199             if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
 200                 print STDERR "skipping $url (unsupported scheme)\n";
 201                 next;
 202             }
 203
 204             next if defined $exclude && $url =~ /$exclude/;
 205
 206             if (alreadyMirrored($algo, $hash)) {
 207                 $have++;
 208                 last;
 209             }
 210
 211             print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
 212
 213             if ($dryRun) {
 214                 $mirrored++;
 215                 last;
 216             }
 217
 218             # Substitute the output.
 219             if (!isValidPath($storePath)) {
 220                 system("nix-store", "-r", $storePath);
 221             }
 222
 223             # Otherwise download the file using nix-prefetch-url.
 224             if (!isValidPath($storePath)) {
 225                 $ENV{QUIET} = 1;
 226                 $ENV{PRINT_PATH} = 1;
 227                 my $fh;
 228                 my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
 229                 waitpid($pid, 0) or die;
 230                 if ($? != 0) {
 231                     print STDERR "failed to fetch $url: $?\n";
 232                     next;
 233                 }
 234                 <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
 235                 if ($storePath ne $storePath2) {
 236                     warn "strange: $storePath != $storePath2\n";
 237                     next;
 238                 }
 239             }
 240
 241             uploadFile($storePath, $url);
 242             $mirrored++;
 243             last;
 244         }
 245     }
 246
 247     print STDERR "mirrored $mirrored files, already have $have files\n";
 248 }
 249
 250 else {
 251     usage();
 252 }