pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs

   1 use anyhow::{anyhow, bail, Context};
   2 use rayon::slice::ParallelSliceMut;
   3 use serde::{
   4     de::{self, Visitor},
   5     Deserialize, Deserializer,
   6 };
   7 use std::{
   8     cmp::Ordering,
   9     collections::{HashMap, HashSet},
  10     fmt,
  11 };
  12 use url::Url;
  13
  14 pub(super) fn packages(content: &str) -> anyhow::Result<Vec<Package>> {
  15     let lockfile: Lockfile = serde_json::from_str(content)?;
  16
  17     let mut packages = match lockfile.version {
  18         1 => {
  19             let initial_url = get_initial_url()?;
  20
  21             to_new_packages(lockfile.dependencies.unwrap_or_default(), &initial_url)?
  22         }
  23         2 | 3 => lockfile
  24             .packages
  25             .unwrap_or_default()
  26             .into_iter()
  27             .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_))))
  28             .map(|(n, p)| Package { name: Some(n), ..p })
  29             .collect(),
  30         _ => bail!(
  31             "We don't support lockfile version {}, please file an issue.",
  32             lockfile.version
  33         ),
  34     };
  35
  36     packages.par_sort_by(|x, y| {
  37         x.resolved
  38             .partial_cmp(&y.resolved)
  39             .expect("resolved should be comparable")
  40             .then(
  41                 // v1 lockfiles can contain multiple references to the same version of a package, with
  42                 // different integrity values (e.g. a SHA-1 and a SHA-512 in one, but just a SHA-512 in another)
  43                 y.integrity
  44                     .partial_cmp(&x.integrity)
  45                     .expect("integrity should be comparable"),
  46             )
  47     });
  48
  49     packages.dedup_by(|x, y| x.resolved == y.resolved);
  50
  51     Ok(packages)
  52 }
  53
  54 #[derive(Deserialize)]
  55 struct Lockfile {
  56     #[serde(rename = "lockfileVersion")]
  57     version: u8,
  58     dependencies: Option<HashMap<String, OldPackage>>,
  59     packages: Option<HashMap<String, Package>>,
  60 }
  61
  62 #[derive(Deserialize)]
  63 struct OldPackage {
  64     version: UrlOrString,
  65     #[serde(default)]
  66     bundled: bool,
  67     resolved: Option<UrlOrString>,
  68     integrity: Option<HashCollection>,
  69     dependencies: Option<HashMap<String, OldPackage>>,
  70 }
  71
  72 #[derive(Debug, Deserialize, PartialEq, Eq)]
  73 pub(super) struct Package {
  74     #[serde(default)]
  75     pub(super) name: Option<String>,
  76     pub(super) resolved: Option<UrlOrString>,
  77     pub(super) integrity: Option<HashCollection>,
  78 }
  79
  80 #[derive(Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
  81 #[serde(untagged)]
  82 pub(super) enum UrlOrString {
  83     Url(Url),
  84     String(String),
  85 }
  86
  87 impl fmt::Display for UrlOrString {
  88     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  89         match self {
  90             UrlOrString::Url(url) => url.fmt(f),
  91             UrlOrString::String(string) => string.fmt(f),
  92         }
  93     }
  94 }
  95
  96 #[derive(Debug, PartialEq, Eq)]
  97 pub struct HashCollection(HashSet<Hash>);
  98
  99 impl HashCollection {
 100     pub fn from_str(s: impl AsRef<str>) -> anyhow::Result<HashCollection> {
 101         let hashes = s
 102             .as_ref()
 103             .split_ascii_whitespace()
 104             .map(Hash::new)
 105             .collect::<anyhow::Result<_>>()?;
 106
 107         Ok(HashCollection(hashes))
 108     }
 109
 110     pub fn into_best(self) -> Option<Hash> {
 111         self.0.into_iter().max()
 112     }
 113 }
 114
 115 impl PartialOrd for HashCollection {
 116     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
 117         let lhs = self.0.iter().max()?;
 118         let rhs = other.0.iter().max()?;
 119
 120         lhs.partial_cmp(rhs)
 121     }
 122 }
 123
 124 impl<'de> Deserialize<'de> for HashCollection {
 125     fn deserialize<D>(deserializer: D) -> Result<HashCollection, D::Error>
 126     where
 127         D: Deserializer<'de>,
 128     {
 129         deserializer.deserialize_string(HashCollectionVisitor)
 130     }
 131 }
 132
 133 struct HashCollectionVisitor;
 134
 135 impl<'de> Visitor<'de> for HashCollectionVisitor {
 136     type Value = HashCollection;
 137
 138     fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
 139         formatter.write_str("a single SRI hash or a collection of them (separated by spaces)")
 140     }
 141
 142     fn visit_str<E>(self, value: &str) -> Result<HashCollection, E>
 143     where
 144         E: de::Error,
 145     {
 146         HashCollection::from_str(value).map_err(E::custom)
 147     }
 148 }
 149
 150 #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
 151 pub struct Hash(String);
 152
 153 // Hash algorithms, in ascending preference.
 154 const ALGOS: &[&str] = &["sha1", "sha512"];
 155
 156 impl Hash {
 157     fn new(s: impl AsRef<str>) -> anyhow::Result<Hash> {
 158         let algo = s
 159             .as_ref()
 160             .split_once('-')
 161             .ok_or_else(|| anyhow!("expected SRI hash, got {:?}", s.as_ref()))?
 162             .0;
 163
 164         if ALGOS.iter().any(|&a| algo == a) {
 165             Ok(Hash(s.as_ref().to_string()))
 166         } else {
 167             Err(anyhow!("unknown hash algorithm {algo:?}"))
 168         }
 169     }
 170
 171     pub fn as_str(&self) -> &str {
 172         &self.0
 173     }
 174 }
 175
 176 impl fmt::Display for Hash {
 177     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 178         self.as_str().fmt(f)
 179     }
 180 }
 181
 182 #[allow(clippy::non_canonical_partial_ord_impl)]
 183 impl PartialOrd for Hash {
 184     fn partial_cmp(&self, other: &Hash) -> Option<Ordering> {
 185         let lhs = self.0.split_once('-')?.0;
 186         let rhs = other.0.split_once('-')?.0;
 187
 188         ALGOS
 189             .iter()
 190             .position(|&s| lhs == s)?
 191             .partial_cmp(&ALGOS.iter().position(|&s| rhs == s)?)
 192     }
 193 }
 194
 195 impl Ord for Hash {
 196     fn cmp(&self, other: &Hash) -> Ordering {
 197         self.partial_cmp(other).unwrap()
 198     }
 199 }
 200
 201 #[allow(clippy::case_sensitive_file_extension_comparisons)]
 202 fn to_new_packages(
 203     old_packages: HashMap<String, OldPackage>,
 204     initial_url: &Url,
 205 ) -> anyhow::Result<Vec<Package>> {
 206     let mut new = Vec::new();
 207
 208     for (name, mut package) in old_packages {
 209         // In some cases, a bundled dependency happens to have the same version as a non-bundled one, causing
 210         // the bundled one without a URL to override the entry for the non-bundled instance, which prevents the
 211         // dependency from being downloaded.
 212         if package.bundled {
 213             continue;
 214         }
 215
 216         if let UrlOrString::Url(v) = &package.version {
 217             if v.scheme() == "npm" {
 218                 if let Some(UrlOrString::Url(ref url)) = &package.resolved {
 219                     package.version = UrlOrString::Url(url.clone());
 220                 }
 221             } else {
 222                 for (scheme, host) in [
 223                     ("github", "github.com"),
 224                     ("bitbucket", "bitbucket.org"),
 225                     ("gitlab", "gitlab.com"),
 226                 ] {
 227                     if v.scheme() == scheme {
 228                         package.version = {
 229                             let mut new_url = initial_url.clone();
 230
 231                             new_url.set_host(Some(host))?;
 232
 233                             if v.path().ends_with(".git") {
 234                                 new_url.set_path(v.path());
 235                             } else {
 236                                 new_url.set_path(&format!("{}.git", v.path()));
 237                             }
 238
 239                             new_url.set_fragment(v.fragment());
 240
 241                             UrlOrString::Url(new_url)
 242                         };
 243
 244                         break;
 245                     }
 246                 }
 247             }
 248         }
 249
 250         new.push(Package {
 251             name: Some(name),
 252             resolved: if matches!(package.version, UrlOrString::Url(_)) {
 253                 Some(package.version)
 254             } else {
 255                 package.resolved
 256             },
 257             integrity: package.integrity,
 258         });
 259
 260         if let Some(dependencies) = package.dependencies {
 261             new.append(&mut to_new_packages(dependencies, initial_url)?);
 262         }
 263     }
 264
 265     Ok(new)
 266 }
 267
 268 fn get_initial_url() -> anyhow::Result<Url> {
 269     Url::parse("git+ssh://git@a.b").context("initial url should be valid")
 270 }
 271
 272 #[cfg(test)]
 273 mod tests {
 274     use super::{
 275         get_initial_url, packages, to_new_packages, Hash, HashCollection, OldPackage, Package,
 276         UrlOrString,
 277     };
 278     use std::{
 279         cmp::Ordering,
 280         collections::{HashMap, HashSet},
 281     };
 282     use url::Url;
 283
 284     #[test]
 285     fn git_shorthand_v1() -> anyhow::Result<()> {
 286         let old = {
 287             let mut o = HashMap::new();
 288             o.insert(
 289                 String::from("sqlite3"),
 290                 OldPackage {
 291                     version: UrlOrString::Url(
 292                         Url::parse(
 293                             "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a",
 294                         )
 295                         .unwrap(),
 296                     ),
 297                     bundled: false,
 298                     resolved: None,
 299                     integrity: None,
 300                     dependencies: None,
 301                 },
 302             );
 303             o
 304         };
 305
 306         let initial_url = get_initial_url()?;
 307
 308         let new = to_new_packages(old, &initial_url)?;
 309
 310         assert_eq!(new.len(), 1, "new packages map should contain 1 value");
 311         assert_eq!(new[0], Package {
 312             name: Some(String::from("sqlite3")),
 313             resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())),
 314             integrity: None
 315         });
 316
 317         Ok(())
 318     }
 319
 320     #[test]
 321     fn hash_preference() {
 322         assert_eq!(
 323             Hash(String::from("sha1-foo")).partial_cmp(&Hash(String::from("sha512-foo"))),
 324             Some(Ordering::Less)
 325         );
 326
 327         assert_eq!(
 328             HashCollection({
 329                 let mut set = HashSet::new();
 330                 set.insert(Hash(String::from("sha512-foo")));
 331                 set.insert(Hash(String::from("sha1-bar")));
 332                 set
 333             })
 334             .into_best(),
 335             Some(Hash(String::from("sha512-foo")))
 336         );
 337     }
 338
 339     #[test]
 340     fn parse_lockfile_correctly() {
 341         let packages = packages(
 342             r#"{
 343                 "name": "node-ddr",
 344                 "version": "1.0.0",
 345                 "lockfileVersion": 1,
 346                 "requires": true,
 347                 "dependencies": {
 348                     "string-width-cjs": {
 349                         "version": "npm:string-width@4.2.3",
 350                         "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
 351                         "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
 352                         "requires": {
 353                             "emoji-regex": "^8.0.0",
 354                             "is-fullwidth-code-point": "^3.0.0",
 355                             "strip-ansi": "^6.0.1"
 356                         }
 357                     }
 358                 }
 359             }"#).unwrap();
 360
 361         assert_eq!(packages.len(), 1);
 362         assert_eq!(
 363             packages[0].resolved,
 364             Some(UrlOrString::Url(
 365                 Url::parse("https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz")
 366                     .unwrap()
 367             ))
 368         );
 369     }
 370 }