pkgs/build-support/node/fetch-npm-deps/src/main.rs

   1 #![warn(clippy::pedantic)]
   2
   3 use crate::cacache::{Cache, Key};
   4 use anyhow::{anyhow, bail};
   5 use rayon::prelude::*;
   6 use serde_json::{Map, Value};
   7 use std::{
   8     collections::HashMap,
   9     env, fs,
  10     path::{Path, PathBuf},
  11     process,
  12 };
  13 use tempfile::tempdir;
  14 use url::Url;
  15 use walkdir::WalkDir;
  16
  17 mod cacache;
  18 mod parse;
  19 mod util;
  20
  21 fn cache_map_path() -> Option<PathBuf> {
  22     env::var_os("CACHE_MAP_PATH").map(PathBuf::from)
  23 }
  24
  25 /// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies.
  26 ///
  27 /// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1
  28 /// and SHA-512) in the lockfile. Given we only cache one version of these, the `integrity` field must be normalized to the hash
  29 /// we cache as (which is the strongest available one).
  30 ///
  31 /// Git dependencies from specific providers can be retrieved from those providers' automatic tarball features.
  32 /// When these dependencies are specified with a commit identifier, npm generates a tarball, and inserts the integrity hash of that
  33 /// tarball into the lockfile.
  34 ///
  35 /// Thus, we remove this hash, to replace it with our own determinstic copies of dependencies from hosted Git providers.
  36 ///
  37 /// If no fixups were performed, `None` is returned and the lockfile structure should be left as-is. If fixups were performed, the
  38 /// `dependencies` key in v2 lockfiles designed for backwards compatibility with v1 parsers is removed because of inconsistent data.
  39 fn fixup_lockfile(
  40     mut lock: Map<String, Value>,
  41     cache: &Option<HashMap<String, String>>,
  42 ) -> anyhow::Result<Option<Map<String, Value>>> {
  43     let mut fixed = false;
  44
  45     match lock
  46         .get("lockfileVersion")
  47         .ok_or_else(|| anyhow!("couldn't get lockfile version"))?
  48         .as_i64()
  49         .ok_or_else(|| anyhow!("lockfile version isn't an int"))?
  50     {
  51         1 => fixup_v1_deps(
  52             lock.get_mut("dependencies")
  53                 .unwrap()
  54                 .as_object_mut()
  55                 .unwrap(),
  56             cache,
  57             &mut fixed,
  58         ),
  59         2 | 3 => {
  60             for package in lock
  61                 .get_mut("packages")
  62                 .ok_or_else(|| anyhow!("couldn't get packages"))?
  63                 .as_object_mut()
  64                 .ok_or_else(|| anyhow!("packages isn't a map"))?
  65                 .values_mut()
  66             {
  67                 if let Some(Value::String(resolved)) = package.get("resolved") {
  68                     if let Some(Value::String(integrity)) = package.get("integrity") {
  69                         if resolved.starts_with("git+ssh://") {
  70                             fixed = true;
  71
  72                             package
  73                                 .as_object_mut()
  74                                 .ok_or_else(|| anyhow!("package isn't a map"))?
  75                                 .remove("integrity");
  76                         } else if let Some(cache_hashes) = cache {
  77                             let cache_hash = cache_hashes
  78                                 .get(resolved)
  79                                 .expect("dependency should have a hash");
  80
  81                             if integrity != cache_hash {
  82                                 fixed = true;
  83
  84                                 *package
  85                                     .as_object_mut()
  86                                     .ok_or_else(|| anyhow!("package isn't a map"))?
  87                                     .get_mut("integrity")
  88                                     .unwrap() = Value::String(cache_hash.clone());
  89                             }
  90                         }
  91                     }
  92                 }
  93             }
  94
  95             if fixed {
  96                 lock.remove("dependencies");
  97             }
  98         }
  99         v => bail!("unsupported lockfile version {v}"),
 100     }
 101
 102     if fixed {
 103         Ok(Some(lock))
 104     } else {
 105         Ok(None)
 106     }
 107 }
 108
 109 // Recursive helper to fixup v1 lockfile deps
 110 fn fixup_v1_deps(
 111     dependencies: &mut Map<String, Value>,
 112     cache: &Option<HashMap<String, String>>,
 113     fixed: &mut bool,
 114 ) {
 115     for dep in dependencies.values_mut() {
 116         if let Some(Value::String(resolved)) = dep
 117             .as_object()
 118             .expect("v1 dep must be object")
 119             .get("resolved")
 120         {
 121             if let Some(Value::String(integrity)) = dep
 122                 .as_object()
 123                 .expect("v1 dep must be object")
 124                 .get("integrity")
 125             {
 126                 if resolved.starts_with("git+ssh://") {
 127                     *fixed = true;
 128
 129                     dep.as_object_mut()
 130                         .expect("v1 dep must be object")
 131                         .remove("integrity");
 132                 } else if let Some(cache_hashes) = cache {
 133                     let cache_hash = cache_hashes
 134                         .get(resolved)
 135                         .expect("dependency should have a hash");
 136
 137                     if integrity != cache_hash {
 138                         *fixed = true;
 139
 140                         *dep.as_object_mut()
 141                             .expect("v1 dep must be object")
 142                             .get_mut("integrity")
 143                             .unwrap() = Value::String(cache_hash.clone());
 144                     }
 145                 }
 146             }
 147         }
 148
 149         if let Some(Value::Object(more_deps)) = dep.as_object_mut().unwrap().get_mut("dependencies")
 150         {
 151             fixup_v1_deps(more_deps, cache, fixed);
 152         }
 153     }
 154 }
 155
 156 fn map_cache() -> anyhow::Result<HashMap<Url, String>> {
 157     let mut hashes = HashMap::new();
 158
 159     let content_path = Path::new(&env::var_os("npmDeps").unwrap()).join("_cacache/index-v5");
 160
 161     for entry in WalkDir::new(content_path) {
 162         let entry = entry?;
 163
 164         if entry.file_type().is_file() {
 165             let content = fs::read_to_string(entry.path())?;
 166             let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?;
 167
 168             hashes.insert(key.metadata.url, key.integrity);
 169         }
 170     }
 171
 172     Ok(hashes)
 173 }
 174
 175 fn main() -> anyhow::Result<()> {
 176     env_logger::init();
 177
 178     let args = env::args().collect::<Vec<_>>();
 179
 180     if args.len() < 2 {
 181         println!("usage: {} <path/to/package-lock.json>", args[0]);
 182         println!();
 183         println!("Prefetches npm dependencies for usage by fetchNpmDeps.");
 184
 185         process::exit(1);
 186     }
 187
 188     if let Ok(jobs) = env::var("NIX_BUILD_CORES") {
 189         if !jobs.is_empty() {
 190             rayon::ThreadPoolBuilder::new()
 191                 .num_threads(
 192                     jobs.parse()
 193                         .expect("NIX_BUILD_CORES must be a whole number"),
 194                 )
 195                 .build_global()
 196                 .unwrap();
 197         }
 198     }
 199
 200     if args[1] == "--fixup-lockfile" {
 201         let lock = serde_json::from_str(&fs::read_to_string(&args[2])?)?;
 202
 203         let cache = cache_map_path()
 204             .map(|map_path| Ok::<_, anyhow::Error>(serde_json::from_slice(&fs::read(map_path)?)?))
 205             .transpose()?;
 206
 207         if let Some(fixed) = fixup_lockfile(lock, &cache)? {
 208             println!("Fixing lockfile");
 209
 210             fs::write(&args[2], serde_json::to_string(&fixed)?)?;
 211         }
 212
 213         return Ok(());
 214     } else if args[1] == "--map-cache" {
 215         let map = map_cache()?;
 216
 217         fs::write(
 218             cache_map_path().expect("CACHE_MAP_PATH environment variable must be set"),
 219             serde_json::to_string(&map)?,
 220         )?;
 221
 222         return Ok(());
 223     }
 224
 225     let lock_content = fs::read_to_string(&args[1])?;
 226
 227     let out_tempdir;
 228
 229     let (out, print_hash) = if let Some(path) = args.get(2) {
 230         (Path::new(path), false)
 231     } else {
 232         out_tempdir = tempdir()?;
 233
 234         (out_tempdir.path(), true)
 235     };
 236
 237     let packages = parse::lockfile(
 238         &lock_content,
 239         env::var("FORCE_GIT_DEPS").is_ok(),
 240         env::var("FORCE_EMPTY_CACHE").is_ok(),
 241     )?;
 242
 243     let cache = Cache::new(out.join("_cacache"));
 244     cache.init()?;
 245
 246     packages.into_par_iter().try_for_each(|package| {
 247         let tarball = package
 248             .tarball()
 249             .map_err(|e| anyhow!("couldn't fetch {} at {}: {e:?}", package.name, package.url))?;
 250         let integrity = package.integrity().map(ToString::to_string);
 251
 252         cache
 253             .put(
 254                 format!("make-fetch-happen:request-cache:{}", package.url),
 255                 package.url,
 256                 &tarball,
 257                 integrity,
 258             )
 259             .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?;
 260
 261         Ok::<_, anyhow::Error>(())
 262     })?;
 263
 264     fs::write(out.join("package-lock.json"), lock_content)?;
 265
 266     if print_hash {
 267         println!("{}", util::make_sri_hash(out)?);
 268     }
 269
 270     Ok(())
 271 }
 272
 273 #[cfg(test)]
 274 mod tests {
 275     use std::collections::HashMap;
 276
 277     use super::fixup_lockfile;
 278     use serde_json::json;
 279
 280     #[test]
 281     fn lockfile_fixup() -> anyhow::Result<()> {
 282         let input = json!({
 283             "lockfileVersion": 2,
 284             "name": "foo",
 285             "packages": {
 286                 "": {
 287
 288                 },
 289                 "foo": {
 290                     "resolved": "https://github.com/NixOS/nixpkgs",
 291                     "integrity": "sha1-aaa"
 292                 },
 293                 "bar": {
 294                     "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
 295                     "integrity": "sha512-aaa"
 296                 },
 297                 "foo-bad": {
 298                     "resolved": "foo",
 299                     "integrity": "sha1-foo"
 300                 },
 301                 "foo-good": {
 302                     "resolved": "foo",
 303                     "integrity": "sha512-foo"
 304                 },
 305             }
 306         });
 307
 308         let expected = json!({
 309             "lockfileVersion": 2,
 310             "name": "foo",
 311             "packages": {
 312                 "": {
 313
 314                 },
 315                 "foo": {
 316                     "resolved": "https://github.com/NixOS/nixpkgs",
 317                     "integrity": ""
 318                 },
 319                 "bar": {
 320                     "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
 321                 },
 322                 "foo-bad": {
 323                     "resolved": "foo",
 324                     "integrity": "sha512-foo"
 325                 },
 326                 "foo-good": {
 327                     "resolved": "foo",
 328                     "integrity": "sha512-foo"
 329                 },
 330             }
 331         });
 332
 333         let mut hashes = HashMap::new();
 334
 335         hashes.insert(
 336             String::from("https://github.com/NixOS/nixpkgs"),
 337             String::new(),
 338         );
 339
 340         hashes.insert(
 341             String::from("git+ssh://git@github.com/NixOS/nixpkgs.git"),
 342             String::new(),
 343         );
 344
 345         hashes.insert(String::from("foo"), String::from("sha512-foo"));
 346
 347         assert_eq!(
 348             fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
 349             Some(expected.as_object().unwrap().clone())
 350         );
 351
 352         Ok(())
 353     }
 354
 355     #[test]
 356     fn lockfile_v1_fixup() -> anyhow::Result<()> {
 357         let input = json!({
 358             "lockfileVersion": 1,
 359             "name": "foo",
 360             "dependencies": {
 361                 "foo": {
 362                     "resolved": "https://github.com/NixOS/nixpkgs",
 363                     "integrity": "sha512-aaa"
 364                 },
 365                 "foo-good": {
 366                     "resolved": "foo",
 367                     "integrity": "sha512-foo"
 368                 },
 369                 "bar": {
 370                     "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
 371                     "integrity": "sha512-bbb",
 372                     "dependencies": {
 373                         "foo-bad": {
 374                             "resolved": "foo",
 375                             "integrity": "sha1-foo"
 376                         },
 377                     },
 378                 },
 379             }
 380         });
 381
 382         let expected = json!({
 383             "lockfileVersion": 1,
 384             "name": "foo",
 385             "dependencies": {
 386                 "foo": {
 387                     "resolved": "https://github.com/NixOS/nixpkgs",
 388                     "integrity": ""
 389                 },
 390                 "foo-good": {
 391                     "resolved": "foo",
 392                     "integrity": "sha512-foo"
 393                 },
 394                 "bar": {
 395                     "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
 396                     "dependencies": {
 397                         "foo-bad": {
 398                             "resolved": "foo",
 399                             "integrity": "sha512-foo"
 400                         },
 401                     },
 402                 },
 403             }
 404         });
 405
 406         let mut hashes = HashMap::new();
 407
 408         hashes.insert(
 409             String::from("https://github.com/NixOS/nixpkgs"),
 410             String::new(),
 411         );
 412
 413         hashes.insert(
 414             String::from("git+ssh://git@github.com/NixOS/nixpkgs.git"),
 415             String::new(),
 416         );
 417
 418         hashes.insert(String::from("foo"), String::from("sha512-foo"));
 419
 420         assert_eq!(
 421             fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
 422             Some(expected.as_object().unwrap().clone())
 423         );
 424
 425         Ok(())
 426     }
 427 }