pkgs/by-name/au/auto-patchelf/source/auto-patchelf.py

   1 #!/usr/bin/env python3
   2
   3 import argparse
   4 import os
   5 import pprint
   6 import subprocess
   7 import sys
   8 import json
   9 from fnmatch import fnmatch
  10 from collections import defaultdict
  11 from contextlib import contextmanager
  12 from dataclasses import dataclass
  13 from itertools import chain
  14 from pathlib import Path, PurePath
  15 from typing import DefaultDict, Generator, Iterator, Optional
  16
  17 from elftools.common.exceptions import ELFError  # type: ignore
  18 from elftools.elf.dynamic import DynamicSection  # type: ignore
  19 from elftools.elf.sections import NoteSection  # type: ignore
  20 from elftools.elf.elffile import ELFFile  # type: ignore
  21 from elftools.elf.enums import ENUM_E_TYPE, ENUM_EI_OSABI  # type: ignore
  22
  23
  24 @contextmanager
  25 def open_elf(path: Path) -> Iterator[ELFFile]:
  26     with path.open('rb') as stream:
  27         yield ELFFile(stream)
  28
  29
  30 def is_static_executable(elf: ELFFile) -> bool:
  31     # Statically linked executables have an ELF type of EXEC but no INTERP.
  32     return (elf.header["e_type"] == 'ET_EXEC'
  33             and not elf.get_section_by_name(".interp"))
  34
  35
  36 def is_dynamic_executable(elf: ELFFile) -> bool:
  37     # We do not require an ELF type of EXEC. This also catches
  38     # position-independent executables, as they typically have an INTERP
  39     # section but their ELF type is DYN.
  40     return bool(elf.get_section_by_name(".interp"))
  41
  42
  43 def get_dependencies(elf: ELFFile) -> list[list[Path]]:
  44     dependencies = []
  45     # This convoluted code is here on purpose. For some reason, using
  46     # elf.get_section_by_name(".dynamic") does not always return an
  47     # instance of DynamicSection, but that is required to call iter_tags
  48     for section in elf.iter_sections():
  49         if isinstance(section, DynamicSection):
  50             for tag in section.iter_tags('DT_NEEDED'):
  51                 dependencies.append([Path(tag.needed)])
  52             break # There is only one dynamic section
  53
  54     return dependencies
  55
  56
  57 def get_dlopen_dependencies(elf: ELFFile) -> list[list[Path]]:
  58     """
  59     Extracts dependencies from the `.note.dlopen` section.
  60     This is a FreeDesktop standard to annotate binaries with libraries that it may `dlopen`.
  61     See https://systemd.io/ELF_DLOPEN_METADATA/
  62     """
  63     dependencies = []
  64     for section in elf.iter_sections():
  65         if not isinstance(section, NoteSection) or section.name != ".note.dlopen":
  66             continue
  67         for note in section.iter_notes():
  68             if note["n_type"] != 0x407C0C0A or note["n_name"] != "FDO":
  69                 continue
  70             note_desc = note["n_desc"]
  71             text = note_desc.decode("utf-8").rstrip("\0")
  72             j = json.loads(text)
  73             for d in j:
  74                 dependencies.append([Path(soname) for soname in d["soname"]])
  75     return dependencies
  76
  77
  78 def get_rpath(elf: ELFFile) -> list[str]:
  79     # This convoluted code is here on purpose. For some reason, using
  80     # elf.get_section_by_name(".dynamic") does not always return an
  81     # instance of DynamicSection, but that is required to call iter_tags
  82     for section in elf.iter_sections():
  83         if isinstance(section, DynamicSection):
  84             for tag in section.iter_tags('DT_RUNPATH'):
  85                 return tag.runpath.split(':')
  86
  87             for tag in section.iter_tags('DT_RPATH'):
  88                 return tag.rpath.split(':')
  89
  90             break # There is only one dynamic section
  91
  92     return []
  93
  94
  95 def get_arch(elf: ELFFile) -> str:
  96     return elf.get_machine_arch()
  97
  98
  99 def get_osabi(elf: ELFFile) -> str:
 100     return elf.header["e_ident"]["EI_OSABI"]
 101
 102
 103 def osabi_are_compatible(wanted: str, got: str) -> bool:
 104     """
 105     Tests whether two OS ABIs are compatible, taking into account the
 106     generally accepted compatibility of SVR4 ABI with other ABIs.
 107     """
 108     if not wanted or not got:
 109         # One of the types couldn't be detected, so as a fallback we'll
 110         # assume they're compatible.
 111         return True
 112
 113     # Generally speaking, the base ABI (0x00), which is represented by
 114     # readelf(1) as "UNIX - System V", indicates broad compatibility
 115     # with other ABIs.
 116     #
 117     # TODO: This isn't always true. For example, some OSes embed ABI
 118     # compatibility into SHT_NOTE sections like .note.tag and
 119     # .note.ABI-tag.  It would be prudent to add these to the detection
 120     # logic to produce better ABI information.
 121     if wanted == 'ELFOSABI_SYSV':
 122         return True
 123
 124     # Similarly here, we should be able to link against a superset of
 125     # features, so even if the target has another ABI, this should be
 126     # fine.
 127     if got == 'ELFOSABI_SYSV':
 128         return True
 129
 130     # Otherwise, we simply return whether the ABIs are identical.
 131     return wanted == got
 132
 133
 134 def glob(path: Path, pattern: str, recursive: bool) -> Iterator[Path]:
 135     if path.is_dir():
 136         return path.rglob(pattern) if recursive else path.glob(pattern)
 137     else:
 138         # path.glob won't return anything if the path is not a directory.
 139         # We extend that behavior by matching the file name against the pattern.
 140         # This allows to pass single files instead of dirs to auto_patchelf,
 141         # for greater control on the files to consider.
 142         return [path] if path.match(pattern) else []
 143
 144
 145 cached_paths: set[Path] = set()
 146 soname_cache: DefaultDict[tuple[str, str], list[tuple[Path, str]]] = defaultdict(list)
 147
 148
 149 def populate_cache(initial: list[Path], recursive: bool =False) -> None:
 150     lib_dirs = list(initial)
 151
 152     while lib_dirs:
 153         lib_dir = lib_dirs.pop(0)
 154
 155         if lib_dir in cached_paths:
 156             continue
 157
 158         cached_paths.add(lib_dir)
 159
 160         for path in glob(lib_dir, "*.so*", recursive):
 161             if not path.is_file():
 162                 continue
 163
 164             # As an optimisation, resolve the symlinks here, as the target is unique
 165             # XXX: (layus, 2022-07-25) is this really an optimisation in all cases ?
 166             # It could make the rpath bigger or break the fragile precedence of $out.
 167             resolved = path.resolve()
 168             # Do not use resolved paths when names do not match
 169             if resolved.name != path.name:
 170                 resolved = path
 171
 172             try:
 173                 with open_elf(path) as elf:
 174                     osabi = get_osabi(elf)
 175                     arch = get_arch(elf)
 176                     rpath = [Path(p) for p in get_rpath(elf)
 177                                      if p and '$ORIGIN' not in p]
 178                     lib_dirs += rpath
 179                     soname_cache[(path.name, arch)].append((resolved.parent, osabi))
 180
 181             except ELFError:
 182                 # Not an ELF file in the right format
 183                 pass
 184
 185
 186 def find_dependency(soname: str, soarch: str, soabi: str) -> Optional[Path]:
 187     for lib, libabi in soname_cache[(soname, soarch)]:
 188         if osabi_are_compatible(soabi, libabi):
 189             return lib
 190     return None
 191
 192
 193 @dataclass
 194 class Dependency:
 195     file: Path              # The file that contains the dependency
 196     name: Path              # The name of the dependency
 197     found: bool = False     # Whether it was found somewhere
 198
 199
 200 def auto_patchelf_file(path: Path, runtime_deps: list[Path], append_rpaths: list[Path] = [], keep_libc: bool = False, extra_args: list[str] = []) -> list[Dependency]:
 201     try:
 202         with open_elf(path) as elf:
 203
 204             if is_static_executable(elf):
 205                 # No point patching these
 206                 print(f"skipping {path} because it is statically linked")
 207                 return []
 208
 209             if elf.num_segments() == 0:
 210                 # no segment (e.g. object file)
 211                 print(f"skipping {path} because it contains no segment")
 212                 return []
 213
 214             file_arch = get_arch(elf)
 215             if interpreter_arch != file_arch:
 216                 # Our target architecture is different than this file's
 217                 # architecture, so skip it.
 218                 print(f"skipping {path} because its architecture ({file_arch})"
 219                       f" differs from target ({interpreter_arch})")
 220                 return []
 221
 222             file_osabi = get_osabi(elf)
 223             if not osabi_are_compatible(interpreter_osabi, file_osabi):
 224                 print(f"skipping {path} because its OS ABI ({file_osabi}) is"
 225                       f" not compatible with target ({interpreter_osabi})")
 226                 return []
 227
 228             file_is_dynamic_executable = is_dynamic_executable(elf)
 229
 230             file_dependencies = get_dependencies(elf) + get_dlopen_dependencies(elf)
 231
 232     except ELFError:
 233         return []
 234
 235     rpath = []
 236     if file_is_dynamic_executable:
 237         print("setting interpreter of", path)
 238         subprocess.run(
 239                 ["patchelf", "--set-interpreter", interpreter_path.as_posix(), path.as_posix()] + extra_args,
 240                 check=True)
 241         rpath += runtime_deps
 242
 243     print("searching for dependencies of", path)
 244     dependencies = []
 245     # Be sure to get the output of all missing dependencies instead of
 246     # failing at the first one, because it's more useful when working
 247     # on a new package where you don't yet know the dependencies.
 248     for dep in file_dependencies:
 249         was_found = False
 250         for candidate in dep:
 251
 252             # This loop determines which candidate for a given
 253             # dependency can be found, and how. There may be multiple
 254             # candidates for a dep because of '.note.dlopen'
 255             # dependencies.
 256             #
 257             # 1. If a candidate is an absolute path, it is already a
 258             #    valid dependency if that path exists, and nothing needs
 259             #    to be done. It should be an error if that path does not exist.
 260             # 2. If a candidate is found within libc, it should be dropped
 261             #    and resolved automatically by the dynamic linker, unless
 262             #    keep_libc is enabled.
 263             # 3. If a candidate is found in our library dependencies, that
 264             #    dependency should be added to rpath.
 265             # 4. If all of the above fail, libc dependencies should still be
 266             #    considered found. This is in contrast to step 2, because
 267             #    enabling keep_libc should allow libc to be found in step 3
 268             #    if possible to preserve its presence in rpath.
 269             #
 270             # These conditions are checked in this order, because #2
 271             # and #3 may both be true. In that case, we still want to
 272             # add the dependency to rpath, as the original binary
 273             # presumably had it and this should be preserved.
 274
 275             is_libc = (libc_lib / candidate).is_file()
 276
 277             if candidate.is_absolute() and candidate.is_file():
 278                 was_found = True
 279                 break
 280             elif is_libc and not keep_libc:
 281                 was_found = True
 282                 break
 283             elif found_dependency := find_dependency(candidate.name, file_arch, file_osabi):
 284                 rpath.append(found_dependency)
 285                 dependencies.append(Dependency(path, candidate, found=True))
 286                 print(f"    {candidate} -> found: {found_dependency}")
 287                 was_found = True
 288                 break
 289             elif is_libc and keep_libc:
 290                 was_found = True
 291                 break
 292
 293         if not was_found:
 294             dep_name = dep[0] if len(dep) == 1 else f"any({', '.join(map(str, dep))})"
 295             dependencies.append(Dependency(path, dep_name, found=False))
 296             print(f"    {dep_name} -> not found!")
 297
 298     rpath.extend(append_rpaths)
 299
 300     # Dedup the rpath
 301     rpath_str = ":".join(dict.fromkeys(map(Path.as_posix, rpath)))
 302
 303     if rpath:
 304         print("setting RPATH to:", rpath_str)
 305         subprocess.run(
 306                 ["patchelf", "--set-rpath", rpath_str, path.as_posix()] + extra_args,
 307                 check=True)
 308
 309     return dependencies
 310
 311
 312 def auto_patchelf(
 313         paths_to_patch: list[Path],
 314         lib_dirs: list[Path],
 315         runtime_deps: list[Path],
 316         recursive: bool = True,
 317         ignore_missing: list[str] = [],
 318         append_rpaths: list[Path] = [],
 319         keep_libc: bool = False,
 320         extra_args: list[str] = []) -> None:
 321
 322     if not paths_to_patch:
 323         sys.exit("No paths to patch, stopping.")
 324
 325     # Add all shared objects of the current output path to the cache,
 326     # before lib_dirs, so that they are chosen first in find_dependency.
 327     populate_cache(paths_to_patch, recursive)
 328     populate_cache(lib_dirs)
 329
 330     dependencies = []
 331     for path in chain.from_iterable(glob(p, '*', recursive) for p in paths_to_patch):
 332         if not path.is_symlink() and path.is_file():
 333             dependencies += auto_patchelf_file(path, runtime_deps, append_rpaths, keep_libc, extra_args)
 334
 335     missing = [dep for dep in dependencies if not dep.found]
 336
 337     # Print a summary of the missing dependencies at the end
 338     print(f"auto-patchelf: {len(missing)} dependencies could not be satisfied")
 339     failure = False
 340     for dep in missing:
 341         for pattern in ignore_missing:
 342             if fnmatch(dep.name.name, pattern):
 343                 print(f"warn: auto-patchelf ignoring missing {dep.name} wanted by {dep.file}")
 344                 break
 345         else:
 346             print(f"error: auto-patchelf could not satisfy dependency {dep.name} wanted by {dep.file}")
 347             failure = True
 348
 349     if failure:
 350         sys.exit('auto-patchelf failed to find all the required dependencies.\n'
 351                  'Add the missing dependencies to --libs or use '
 352                  '`--ignore-missing="foo.so.1 bar.so etc.so"`.')
 353
 354
 355 def main() -> None:
 356     parser = argparse.ArgumentParser(
 357         prog="auto-patchelf",
 358         description='auto-patchelf tries as hard as possible to patch the'
 359                     ' provided binary files by looking for compatible'
 360                     'libraries in the provided paths.')
 361     parser.add_argument(
 362         "--ignore-missing",
 363         nargs="*",
 364         type=str,
 365         help="Do not fail when some dependencies are not found.")
 366     parser.add_argument(
 367         "--no-recurse",
 368         dest="recursive",
 369         action="store_false",
 370         help="Disable the recursive traversal of paths to patch.")
 371     parser.add_argument(
 372         "--paths", nargs="*", type=Path,
 373         help="Paths whose content needs to be patched."
 374              " Single files and directories are accepted."
 375              " Directories are traversed recursively by default.")
 376     parser.add_argument(
 377         "--libs", nargs="*", type=Path,
 378         help="Paths where libraries are searched for."
 379              " Single files and directories are accepted."
 380              " Directories are not searched recursively.")
 381     parser.add_argument(
 382         "--runtime-dependencies", nargs="*", type=Path,
 383         help="Paths to prepend to the runtime path of executable binaries."
 384              " Subject to deduplication, which may imply some reordering.")
 385     parser.add_argument(
 386         "--append-rpaths",
 387         nargs="*",
 388         type=Path,
 389         help="Paths to append to all runtime paths unconditionally",
 390     )
 391     parser.add_argument(
 392         "--keep-libc",
 393         dest="keep_libc",
 394         action="store_true",
 395         help="Attempt to search for and relink libc dependencies.",
 396     )
 397     parser.add_argument(
 398         "--extra-args",
 399         # Undocumented Python argparse feature: consume all remaining arguments
 400         # as values for this one. This means this argument should always be passed
 401         # last.
 402         nargs="...",
 403         type=str,
 404         help="Extra arguments to pass to patchelf. This argument should always come last."
 405     )
 406
 407     print("automatically fixing dependencies for ELF files")
 408     args = parser.parse_args()
 409     pprint.pprint(vars(args))
 410
 411     auto_patchelf(
 412         args.paths,
 413         args.libs,
 414         args.runtime_dependencies,
 415         args.recursive,
 416         args.ignore_missing,
 417         append_rpaths=args.append_rpaths,
 418         keep_libc=args.keep_libc,
 419         extra_args=args.extra_args)
 420
 421
 422 interpreter_path: Path  = None # type: ignore
 423 interpreter_osabi: str  = None # type: ignore
 424 interpreter_arch: str   = None # type: ignore
 425 libc_lib: Path          = None # type: ignore
 426
 427 if __name__ == "__main__":
 428     nix_support = Path(os.environ.get('NIX_BINTOOLS', os.environ['DEFAULT_BINTOOLS'])) / 'nix-support'
 429     interpreter_path = Path((nix_support / 'dynamic-linker').read_text().strip())
 430     libc_lib = Path((nix_support / 'orig-libc').read_text().strip()) / 'lib'
 431
 432     with open_elf(interpreter_path) as interpreter:
 433         interpreter_osabi = get_osabi(interpreter)
 434         interpreter_arch = get_arch(interpreter)
 435
 436     if interpreter_arch and interpreter_osabi and interpreter_path and libc_lib:
 437         main()
 438     else:
 439         sys.exit("Failed to parse dynamic linker (ld) properties.")