pkgs/build-support/node/fetch-yarn-deps/index.js

   1 #!/usr/bin/env node
   2 'use strict'
   3
   4 const fs = require('fs')
   5 const crypto = require('crypto')
   6 const process = require('process')
   7 const https = require('https')
   8 const child_process = require('child_process')
   9 const path = require('path')
  10 const lockfile = require('./yarnpkg-lockfile.js')
  11 const { promisify } = require('util')
  12 const url = require('url')
  13 const { urlToName } = require('./common.js')
  14
  15 const execFile = promisify(child_process.execFile)
  16
  17 const exec = async (...args) => {
  18         const res = await execFile(...args)
  19         if (res.error) throw new Error(res.stderr)
  20         return res
  21 }
  22
  23 const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => {
  24         return new Promise((resolve, reject) => {
  25                 const get = (url, redirects = 0) => https.get(url, (res) => {
  26                         if(redirects > 10) {
  27                                 reject('Too many redirects!');
  28                                 return;
  29                         }
  30                         if(res.statusCode === 301 || res.statusCode === 302) {
  31                                 return get(res.headers.location, redirects + 1)
  32                         }
  33                         const file = fs.createWriteStream(fileName)
  34                         const hash = crypto.createHash(hashType)
  35                         res.pipe(file)
  36                         res.pipe(hash).setEncoding('hex')
  37                         res.on('end', () => {
  38                                 file.close()
  39                                 const h = hash.read()
  40                                 if (expectedHash === undefined){
  41                                         console.log(`Warning: lockfile url ${url} doesn't end in "#<hash>" to validate against. Downloaded file had hash ${h}.`);
  42                                 } else if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h} for ${url}`))
  43                                 resolve()
  44                         })
  45                         res.on('error', e => reject(e))
  46                 })
  47                 get(url)
  48         })
  49 }
  50
  51 const downloadGit = async (fileName, url, rev) => {
  52         await exec('nix-prefetch-git', [
  53                 '--out', fileName + '.tmp',
  54                 '--url', url,
  55                 '--rev', rev,
  56                 '--builder'
  57         ])
  58
  59         await exec('tar', [
  60                 // hopefully make it reproducible across runs and systems
  61                 '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1',
  62
  63                 // Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79
  64                 '--mode', 'u+w',
  65
  66                 '-C', fileName + '.tmp',
  67                 '-cf', fileName, '.'
  68         ])
  69
  70         await exec('rm', [ '-rf', fileName + '.tmp', ])
  71 }
  72
  73 const isGitUrl = pattern => {
  74         // https://github.com/yarnpkg/yarn/blob/3119382885ea373d3c13d6a846de743eca8c914b/src/resolvers/exotics/git-resolver.js#L15-L47
  75         const GIT_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org']
  76         const GIT_PATTERN_MATCHERS = [/^git:/, /^git\+.+:/, /^ssh:/, /^https?:.+\.git$/, /^https?:.+\.git#.+/]
  77
  78         for (const matcher of GIT_PATTERN_MATCHERS) if (matcher.test(pattern)) return true
  79
  80         const {hostname, path} = url.parse(pattern)
  81         if (hostname && path && GIT_HOSTS.indexOf(hostname) >= 0
  82                 // only if dependency is pointing to a git repo,
  83                 // e.g. facebook/flow and not file in a git repo facebook/flow/archive/v1.0.0.tar.gz
  84                 && path.split('/').filter(p => !!p).length === 2
  85         ) return true
  86
  87         return false
  88 }
  89
  90 const downloadPkg = (pkg, verbose) => {
  91         for (let marker of ['@file:', '@link:']) {
  92                 const split = pkg.key.split(marker)
  93                 if (split.length == 2) {
  94                         console.info(`ignoring lockfile entry "${split[0]}" which points at path "${split[1]}"`)
  95                         return
  96                 } else if (split.length > 2) {
  97                         throw new Error(`The lockfile entry key "${pkg.key}" contains "${marker}" more than once. Processing is not implemented.`)
  98                 }
  99         }
 100
 101         if (pkg.resolved === undefined) {
 102                 throw new Error(`The lockfile entry with key "${pkg.key}" cannot be downloaded because it is missing the "resolved" attribute, which should contain the URL to download from. The lockfile might be invalid.`)
 103         }
 104
 105         const [ url, hash ] = pkg.resolved.split('#')
 106         if (verbose) console.log('downloading ' + url)
 107         const fileName = urlToName(url)
 108         const s = url.split('/')
 109         if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) {
 110                 return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1])
 111         } else if (url.startsWith('https://github.com/') && url.endsWith('.tar.gz') &&
 112                 (
 113                         s.length <= 5 ||    // https://github.com/owner/repo.tgz#feedface...
 114                         s[5] == "archive"   // https://github.com/owner/repo/archive/refs/tags/v0.220.1.tar.gz
 115                 )) {
 116                 return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1].replace(/.tar.gz$/, ''))
 117         } else if (isGitUrl(url)) {
 118                 return downloadGit(fileName, url.replace(/^git\+/, ''), hash)
 119         } else if (url.startsWith('https://')) {
 120                 if (typeof pkg.integrity === 'string' || pkg.integrity instanceof String) {
 121                         const [ type, checksum ] = pkg.integrity.split('-')
 122                         return downloadFileHttps(fileName, url, Buffer.from(checksum, 'base64').toString('hex'), type)
 123                 }
 124                 return downloadFileHttps(fileName, url, hash)
 125         } else if (url.startsWith('file:')) {
 126                 console.warn(`ignoring unsupported file:path url "${url}"`)
 127         } else {
 128                 throw new Error('don\'t know how to download "' + url + '"')
 129         }
 130 }
 131
 132 const performParallel = tasks => {
 133         const worker = async () => {
 134                 while (tasks.length > 0) await tasks.shift()()
 135         }
 136
 137         const workers = []
 138         for (let i = 0; i < 4; i++) {
 139                 workers.push(worker())
 140         }
 141
 142         return Promise.all(workers)
 143 }
 144
 145 // This could be implemented using [`Map.groupBy`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/groupBy),
 146 // but that method is only supported starting with Node 21
 147 const uniqueBy = (arr, callback) => {
 148         const map = new Map()
 149         for (const elem of arr) {
 150                 map.set(callback(elem), elem)
 151         }
 152         return [...map.values()]
 153 }
 154
 155 const prefetchYarnDeps = async (lockContents, verbose) => {
 156         const lockData = lockfile.parse(lockContents)
 157         await performParallel(
 158                 uniqueBy(Object.entries(lockData.object), ([_, value]) => value.resolved)
 159                 .map(([key, value]) => () => downloadPkg({ key, ...value }, verbose))
 160         )
 161         await fs.promises.writeFile('yarn.lock', lockContents)
 162         if (verbose) console.log('Done')
 163 }
 164
 165 const showUsage = async () => {
 166         process.stderr.write(`
 167 syntax: prefetch-yarn-deps [path to yarn.lock] [options]
 168
 169 Options:
 170   -h --help         Show this help
 171   -v --verbose      Verbose output
 172   --builder         Only perform the download to current directory, then exit
 173 `)
 174         process.exit(1)
 175 }
 176
 177 const main = async () => {
 178         const args = process.argv.slice(2)
 179         let next, lockFile, verbose, isBuilder
 180         while (next = args.shift()) {
 181                 if (next == '--builder') {
 182                         isBuilder = true
 183                 } else if (next == '--verbose' || next == '-v') {
 184                         verbose = true
 185                 } else if (next == '--help' || next == '-h') {
 186                         showUsage()
 187                 } else if (!lockFile) {
 188                         lockFile = next
 189                 } else {
 190                         showUsage()
 191                 }
 192         }
 193         let lockContents
 194         try {
 195                 lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
 196         } catch {
 197                 showUsage()
 198         }
 199
 200         if (isBuilder) {
 201                 await prefetchYarnDeps(lockContents, verbose)
 202         } else {
 203                 const { stdout: tmpDir } = await exec('mktemp', [ '-d' ])
 204
 205                 try {
 206                         process.chdir(tmpDir.trim())
 207                         await prefetchYarnDeps(lockContents, verbose)
 208                         const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ])
 209                         console.log(hash)
 210                 } finally {
 211                         await exec('rm', [ '-rf', tmpDir.trim() ])
 212                 }
 213         }
 214 }
 215
 216 main()
 217         .catch(e => {
 218                 console.error(e)
 219                 process.exit(1)
 220         })