yt_dlp/update.py

   1 from __future__ import annotations
   2
   3 import atexit
   4 import contextlib
   5 import functools
   6 import hashlib
   7 import json
   8 import os
   9 import platform
  10 import re
  11 import subprocess
  12 import sys
  13 from dataclasses import dataclass
  14 from zipimport import zipimporter
  15
  16 from .networking import Request
  17 from .networking.exceptions import HTTPError, network_exceptions
  18 from .utils import (
  19     NO_DEFAULT,
  20     Popen,
  21     deprecation_warning,
  22     format_field,
  23     remove_end,
  24     shell_quote,
  25     system_identifier,
  26     version_tuple,
  27 )
  28 from .version import (
  29     CHANNEL,
  30     ORIGIN,
  31     RELEASE_GIT_HEAD,
  32     UPDATE_HINT,
  33     VARIANT,
  34     __version__,
  35 )
  36
  37 UPDATE_SOURCES = {
  38     'stable': 'yt-dlp/yt-dlp',
  39     'nightly': 'yt-dlp/yt-dlp-nightly-builds',
  40     'master': 'yt-dlp/yt-dlp-master-builds',
  41 }
  42 REPOSITORY = UPDATE_SOURCES['stable']
  43 _INVERSE_UPDATE_SOURCES = {value: key for key, value in UPDATE_SOURCES.items()}
  44
  45 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
  46 _HASH_PATTERN = r'[\da-f]{40}'
  47 _COMMIT_RE = re.compile(rf'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
  48
  49 API_BASE_URL = 'https://api.github.com/repos'
  50
  51 # Backwards compatibility variables for the current channel
  52 API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
  53
  54
  55 @functools.cache
  56 def _get_variant_and_executable_path():
  57     """@returns (variant, executable_path)"""
  58     if getattr(sys, 'frozen', False):
  59         path = sys.executable
  60         if not hasattr(sys, '_MEIPASS'):
  61             return 'py2exe', path
  62         elif sys._MEIPASS == os.path.dirname(path):
  63             return f'{sys.platform}_dir', path
  64         elif sys.platform == 'darwin':
  65             machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else ''
  66         else:
  67             machine = f'_{platform.machine().lower()}'
  68             is_64bits = sys.maxsize > 2**32
  69             # Ref: https://en.wikipedia.org/wiki/Uname#Examples
  70             if machine[1:] in ('x86', 'x86_64', 'amd64', 'i386', 'i686'):
  71                 machine = '_x86' if not is_64bits else ''
  72             # platform.machine() on 32-bit raspbian OS may return 'aarch64', so check "64-bitness"
  73             # See: https://github.com/yt-dlp/yt-dlp/issues/11813
  74             elif machine[1:] == 'aarch64' and not is_64bits:
  75                 machine = '_armv7l'
  76             # sys.executable returns a /tmp/ path for staticx builds (linux_static)
  77             # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information
  78             if static_exe_path := os.getenv('STATICX_PROG_PATH'):
  79                 path = static_exe_path
  80         return f'{remove_end(sys.platform, "32")}{machine}_exe', path
  81
  82     path = os.path.dirname(__file__)
  83     if isinstance(__loader__, zipimporter):
  84         return 'zip', os.path.join(path, '..')
  85     elif (os.path.basename(sys.argv[0]) in ('__main__.py', '-m')
  86           and os.path.exists(os.path.join(path, '../.git/HEAD'))):
  87         return 'source', path
  88     return 'unknown', path
  89
  90
  91 def detect_variant():
  92     return VARIANT or _get_variant_and_executable_path()[0]
  93
  94
  95 @functools.cache
  96 def current_git_head():
  97     if detect_variant() != 'source':
  98         return
  99     with contextlib.suppress(Exception):
 100         stdout, _, _ = Popen.run(
 101             ['git', 'rev-parse', '--short', 'HEAD'],
 102             text=True, cwd=os.path.dirname(os.path.abspath(__file__)),
 103             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 104         if re.fullmatch('[0-9a-f]+', stdout.strip()):
 105             return stdout.strip()
 106
 107
 108 _FILE_SUFFIXES = {
 109     'zip': '',
 110     'win_exe': '.exe',
 111     'win_x86_exe': '_x86.exe',
 112     'darwin_exe': '_macos',
 113     'darwin_legacy_exe': '_macos_legacy',
 114     'linux_exe': '_linux',
 115     'linux_aarch64_exe': '_linux_aarch64',
 116     'linux_armv7l_exe': '_linux_armv7l',
 117 }
 118
 119 _NON_UPDATEABLE_REASONS = {
 120     **{variant: None for variant in _FILE_SUFFIXES},  # Updatable
 121     **{variant: f'Auto-update is not supported for unpackaged {name} executable; Re-download the latest release'
 122        for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS', 'linux_dir': 'Linux'}.items()},
 123     'py2exe': 'py2exe is no longer supported by yt-dlp; This executable cannot be updated',
 124     'source': 'You cannot update when running from source code; Use git to pull the latest changes',
 125     'unknown': 'You installed yt-dlp from a manual build or with a package manager; Use that to update',
 126     'other': 'You are using an unofficial build of yt-dlp; Build the executable again',
 127 }
 128
 129
 130 def is_non_updateable():
 131     if UPDATE_HINT:
 132         return UPDATE_HINT
 133     return _NON_UPDATEABLE_REASONS.get(
 134         detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
 135
 136
 137 def _get_binary_name():
 138     return format_field(_FILE_SUFFIXES, detect_variant(), template='yt-dlp%s', ignore=None, default=None)
 139
 140
 141 def _get_system_deprecation():
 142     MIN_SUPPORTED, MIN_RECOMMENDED = (3, 9), (3, 9)
 143
 144     if sys.version_info > MIN_RECOMMENDED:
 145         return None
 146
 147     major, minor = sys.version_info[:2]
 148     PYTHON_MSG = f'Please update to Python {".".join(map(str, MIN_RECOMMENDED))} or above'
 149
 150     if sys.version_info < MIN_SUPPORTED:
 151         return f'Python version {major}.{minor} is no longer supported! {PYTHON_MSG}'
 152
 153     return f'Support for Python version {major}.{minor} has been deprecated. {PYTHON_MSG}'
 154
 155
 156 def _sha256_file(path):
 157     h = hashlib.sha256()
 158     mv = memoryview(bytearray(128 * 1024))
 159     with open(os.path.realpath(path), 'rb', buffering=0) as f:
 160         for n in iter(lambda: f.readinto(mv), 0):
 161             h.update(mv[:n])
 162     return h.hexdigest()
 163
 164
 165 def _make_label(origin, tag, version=None):
 166     if '/' in origin:
 167         channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
 168     else:
 169         channel = origin
 170     label = f'{channel}@{tag}'
 171     if version and version != tag:
 172         label += f' build {version}'
 173     if channel != origin:
 174         label += f' from {origin}'
 175     return label
 176
 177
 178 @dataclass
 179 class UpdateInfo:
 180     """
 181     Update target information
 182
 183     Can be created by `query_update()` or manually.
 184
 185     Attributes:
 186         tag                 The release tag that will be updated to. If from query_update,
 187                             the value is after API resolution and update spec processing.
 188                             The only property that is required.
 189         version             The actual numeric version (if available) of the binary to be updated to,
 190                             after API resolution and update spec processing. (default: None)
 191         requested_version   Numeric version of the binary being requested (if available),
 192                             after API resolution only. (default: None)
 193         commit              Commit hash (if available) of the binary to be updated to,
 194                             after API resolution and update spec processing. (default: None)
 195                             This value will only match the RELEASE_GIT_HEAD of prerelease builds.
 196         binary_name         Filename of the binary to be updated to. (default: current binary name)
 197         checksum            Expected checksum (if available) of the binary to be
 198                             updated to. (default: None)
 199     """
 200     tag: str
 201     version: str | None = None
 202     requested_version: str | None = None
 203     commit: str | None = None
 204
 205     binary_name: str | None = _get_binary_name()  # noqa: RUF009: Always returns the same value
 206     checksum: str | None = None
 207
 208
 209 class Updater:
 210     # XXX: use class variables to simplify testing
 211     _channel = CHANNEL
 212     _origin = ORIGIN
 213     _update_sources = UPDATE_SOURCES
 214
 215     def __init__(self, ydl, target: str | None = None):
 216         self.ydl = ydl
 217         # For backwards compat, target needs to be treated as if it could be None
 218         self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
 219         # Check if requested_tag is actually the requested repo/channel
 220         if not sep and ('/' in self.requested_tag or self.requested_tag in self._update_sources):
 221             self.requested_channel = self.requested_tag
 222             self.requested_tag: str = None  # type: ignore (we set it later)
 223         elif not self.requested_channel:
 224             # User did not specify a channel, so we are requesting the default channel
 225             self.requested_channel = self._channel.partition('@')[0]
 226
 227         # --update should not be treated as an exact tag request even if CHANNEL has a @tag
 228         self._exact = bool(target) and target != self._channel
 229         if not self.requested_tag:
 230             # User did not specify a tag, so we request 'latest' and track that no exact tag was passed
 231             self.requested_tag = 'latest'
 232             self._exact = False
 233
 234         if '/' in self.requested_channel:
 235             # requested_channel is actually a repository
 236             self.requested_repo = self.requested_channel
 237             if not self.requested_repo.startswith('yt-dlp/') and self.requested_repo != self._origin:
 238                 self.ydl.report_warning(
 239                     f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
 240                     f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}. '
 241                     f'Run {self.ydl._format_err("at your own risk", "light red")}')
 242                 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
 243         else:
 244             # Check if requested_channel resolves to a known repository or else raise
 245             self.requested_repo = self._update_sources.get(self.requested_channel)
 246             if not self.requested_repo:
 247                 self._report_error(
 248                     f'Invalid update channel {self.requested_channel!r} requested. '
 249                     f'Valid channels are {", ".join(self._update_sources)}', True)
 250
 251         self._identifier = f'{detect_variant()} {system_identifier()}'
 252
 253     @property
 254     def current_version(self):
 255         """Current version"""
 256         return __version__
 257
 258     @property
 259     def current_commit(self):
 260         """Current commit hash"""
 261         return RELEASE_GIT_HEAD
 262
 263     def _download_asset(self, name, tag=None):
 264         if not tag:
 265             tag = self.requested_tag
 266
 267         path = 'latest/download' if tag == 'latest' else f'download/{tag}'
 268         url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
 269         self.ydl.write_debug(f'Downloading {name} from {url}')
 270         return self.ydl.urlopen(url).read()
 271
 272     def _call_api(self, tag):
 273         tag = f'tags/{tag}' if tag != 'latest' else tag
 274         url = f'{API_BASE_URL}/{self.requested_repo}/releases/{tag}'
 275         self.ydl.write_debug(f'Fetching release info: {url}')
 276         return json.loads(self.ydl.urlopen(Request(url, headers={
 277             'Accept': 'application/vnd.github+json',
 278             'User-Agent': 'yt-dlp',
 279             'X-GitHub-Api-Version': '2022-11-28',
 280         })).read().decode())
 281
 282     def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
 283         if _VERSION_RE.fullmatch(tag):
 284             return tag, None
 285
 286         api_info = self._call_api(tag)
 287
 288         if tag == 'latest':
 289             requested_version = api_info['tag_name']
 290         else:
 291             match = re.search(rf'\s+(?P<version>{_VERSION_RE.pattern})$', api_info.get('name', ''))
 292             requested_version = match.group('version') if match else None
 293
 294         if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish', '')):
 295             target_commitish = api_info['target_commitish']
 296         else:
 297             match = _COMMIT_RE.match(api_info.get('body', ''))
 298             target_commitish = match.group('hash') if match else None
 299
 300         if not (requested_version or target_commitish):
 301             self._report_error('One of either version or commit hash must be available on the release', expected=True)
 302
 303         return requested_version, target_commitish
 304
 305     def _download_update_spec(self, source_tags):
 306         for tag in source_tags:
 307             try:
 308                 return self._download_asset('_update_spec', tag=tag).decode()
 309             except network_exceptions as error:
 310                 if isinstance(error, HTTPError) and error.status == 404:
 311                     continue
 312                 self._report_network_error(f'fetch update spec: {error}')
 313                 return None
 314
 315         self._report_error(
 316             f'The requested tag {self.requested_tag} does not exist for {self.requested_repo}', True)
 317         return None
 318
 319     def _process_update_spec(self, lockfile: str, resolved_tag: str):
 320         lines = lockfile.splitlines()
 321         is_version2 = any(line.startswith('lockV2 ') for line in lines)
 322
 323         for line in lines:
 324             if is_version2:
 325                 if not line.startswith(f'lockV2 {self.requested_repo} '):
 326                     continue
 327                 _, _, tag, pattern = line.split(' ', 3)
 328             else:
 329                 if not line.startswith('lock '):
 330                     continue
 331                 _, tag, pattern = line.split(' ', 2)
 332
 333             if re.match(pattern, self._identifier):
 334                 if _VERSION_RE.fullmatch(tag):
 335                     if not self._exact:
 336                         return tag
 337                     elif self._version_compare(tag, resolved_tag):
 338                         return resolved_tag
 339                 elif tag != resolved_tag:
 340                     continue
 341
 342                 self._report_error(
 343                     f'yt-dlp cannot be updated to {resolved_tag} since you are on an older Python version '
 344                     'or your operating system is not compatible with the requested build', True)
 345                 return None
 346
 347         return resolved_tag
 348
 349     def _version_compare(self, a: str, b: str):
 350         """
 351         Compare two version strings
 352
 353         This function SHOULD NOT be called if self._exact == True
 354         """
 355         if _VERSION_RE.fullmatch(f'{a}.{b}'):
 356             return version_tuple(a) >= version_tuple(b)
 357         return a == b
 358
 359     def query_update(self, *, _output=False) -> UpdateInfo | None:
 360         """Fetches info about the available update
 361         @returns   An `UpdateInfo` if there is an update available, else None
 362         """
 363         if not self.requested_repo:
 364             self._report_error('No target repository could be determined from input')
 365             return None
 366
 367         try:
 368             requested_version, target_commitish = self._get_version_info(self.requested_tag)
 369         except network_exceptions as e:
 370             self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
 371             return None
 372
 373         if self._exact and self._origin != self.requested_repo:
 374             has_update = True
 375         elif requested_version:
 376             if self._exact:
 377                 has_update = self.current_version != requested_version
 378             else:
 379                 has_update = not self._version_compare(self.current_version, requested_version)
 380         elif target_commitish:
 381             has_update = target_commitish != self.current_commit
 382         else:
 383             has_update = False
 384
 385         resolved_tag = requested_version if self.requested_tag == 'latest' else self.requested_tag
 386         current_label = _make_label(self._origin, self._channel.partition('@')[2] or self.current_version, self.current_version)
 387         requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
 388         latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version: {requested_label}'
 389         if not has_update:
 390             if _output:
 391                 self.ydl.to_screen(f'{latest_or_requested}\nyt-dlp is up to date ({current_label})')
 392             return None
 393
 394         update_spec = self._download_update_spec(('latest', None) if requested_version else (None,))
 395         if not update_spec:
 396             return None
 397         # `result_` prefixed vars == post-_process_update_spec() values
 398         result_tag = self._process_update_spec(update_spec, resolved_tag)
 399         if not result_tag or result_tag == self.current_version:
 400             return None
 401         elif result_tag == resolved_tag:
 402             result_version = requested_version
 403         elif _VERSION_RE.fullmatch(result_tag):
 404             result_version = result_tag
 405         else:  # actual version being updated to is unknown
 406             result_version = None
 407
 408         checksum = None
 409         # Non-updateable variants can get update_info but need to skip checksum
 410         if not is_non_updateable():
 411             try:
 412                 hashes = self._download_asset('SHA2-256SUMS', result_tag)
 413             except network_exceptions as error:
 414                 if not isinstance(error, HTTPError) or error.status != 404:
 415                     self._report_network_error(f'fetch checksums: {error}')
 416                     return None
 417                 self.ydl.report_warning('No hash information found for the release, skipping verification')
 418             else:
 419                 for ln in hashes.decode().splitlines():
 420                     if ln.endswith(_get_binary_name()):
 421                         checksum = ln.split()[0]
 422                         break
 423                 if not checksum:
 424                     self.ydl.report_warning('The hash could not be found in the checksum file, skipping verification')
 425
 426         if _output:
 427             update_label = _make_label(self.requested_repo, result_tag, result_version)
 428             self.ydl.to_screen(
 429                 f'Current version: {current_label}\n{latest_or_requested}'
 430                 + (f'\nUpgradable to: {update_label}' if update_label != requested_label else ''))
 431
 432         return UpdateInfo(
 433             tag=result_tag,
 434             version=result_version,
 435             requested_version=requested_version,
 436             commit=target_commitish if result_tag == resolved_tag else None,
 437             checksum=checksum)
 438
 439     def update(self, update_info=NO_DEFAULT):
 440         """Update yt-dlp executable to the latest version
 441         @param update_info  `UpdateInfo | None` as returned by query_update()
 442         """
 443         if update_info is NO_DEFAULT:
 444             update_info = self.query_update(_output=True)
 445         if not update_info:
 446             return False
 447
 448         err = is_non_updateable()
 449         if err:
 450             self._report_error(err, True)
 451             return False
 452
 453         self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
 454
 455         update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
 456         self.ydl.to_screen(f'Updating to {update_label} ...')
 457
 458         directory = os.path.dirname(self.filename)
 459         if not os.access(self.filename, os.W_OK):
 460             return self._report_permission_error(self.filename)
 461         elif not os.access(directory, os.W_OK):
 462             return self._report_permission_error(directory)
 463
 464         new_filename, old_filename = f'{self.filename}.new', f'{self.filename}.old'
 465         if detect_variant() == 'zip':  # Can be replaced in-place
 466             new_filename, old_filename = self.filename, None
 467
 468         try:
 469             if os.path.exists(old_filename or ''):
 470                 os.remove(old_filename)
 471         except OSError:
 472             return self._report_error('Unable to remove the old version')
 473
 474         try:
 475             newcontent = self._download_asset(update_info.binary_name, update_info.tag)
 476         except network_exceptions as e:
 477             if isinstance(e, HTTPError) and e.status == 404:
 478                 return self._report_error(
 479                     f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
 480             return self._report_network_error(f'fetch updates: {e}', tag=update_info.tag)
 481
 482         if not update_info.checksum:
 483             self._block_restart('Automatically restarting into unverified builds is disabled for security reasons')
 484         elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
 485             return self._report_network_error('verify the new executable', tag=update_info.tag)
 486
 487         try:
 488             with open(new_filename, 'wb') as outf:
 489                 outf.write(newcontent)
 490         except OSError:
 491             return self._report_permission_error(new_filename)
 492
 493         if old_filename:
 494             mask = os.stat(self.filename).st_mode
 495             try:
 496                 os.rename(self.filename, old_filename)
 497             except OSError:
 498                 return self._report_error('Unable to move current version')
 499
 500             try:
 501                 os.rename(new_filename, self.filename)
 502             except OSError:
 503                 self._report_error('Unable to overwrite current version')
 504                 return os.rename(old_filename, self.filename)
 505
 506         variant = detect_variant()
 507         if variant.startswith('win'):
 508             atexit.register(Popen, f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{old_filename}"',
 509                             shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 510         elif old_filename:
 511             try:
 512                 os.remove(old_filename)
 513             except OSError:
 514                 self._report_error('Unable to remove the old version')
 515
 516             try:
 517                 os.chmod(self.filename, mask)
 518             except OSError:
 519                 return self._report_error(
 520                     f'Unable to set permissions. Run: sudo chmod a+rx {shell_quote(self.filename)}')
 521
 522         self.ydl.to_screen(f'Updated yt-dlp to {update_label}')
 523         return True
 524
 525     @functools.cached_property
 526     def filename(self):
 527         """Filename of the executable"""
 528         return os.path.realpath(_get_variant_and_executable_path()[1])
 529
 530     @functools.cached_property
 531     def cmd(self):
 532         """The command-line to run the executable, if known"""
 533         argv = None
 534         # There is no sys.orig_argv in py < 3.10. Also, it can be [] when frozen
 535         if getattr(sys, 'orig_argv', None):
 536             argv = sys.orig_argv
 537         elif getattr(sys, 'frozen', False):
 538             argv = sys.argv
 539         # linux_static exe's argv[0] will be /tmp/staticx-NNNN/yt-dlp_linux if we don't fixup here
 540         if argv and os.getenv('STATICX_PROG_PATH'):
 541             argv = [self.filename, *argv[1:]]
 542         return argv
 543
 544     def restart(self):
 545         """Restart the executable"""
 546         assert self.cmd, 'Must be frozen or Py >= 3.10'
 547         self.ydl.write_debug(f'Restarting: {shell_quote(self.cmd)}')
 548         _, _, returncode = Popen.run(self.cmd)
 549         return returncode
 550
 551     def _block_restart(self, msg):
 552         def wrapper():
 553             self._report_error(f'{msg}. Restart yt-dlp to use the updated version', expected=True)
 554             return self.ydl._download_retcode
 555         self.restart = wrapper
 556
 557     def _report_error(self, msg, expected=False):
 558         self.ydl.report_error(msg, tb=False if expected else None)
 559         self.ydl._download_retcode = 100
 560
 561     def _report_permission_error(self, file):
 562         self._report_error(f'Unable to write to {file}; try running as administrator', True)
 563
 564     def _report_network_error(self, action, delim=';', tag=None):
 565         if not tag:
 566             tag = self.requested_tag
 567         path = tag if tag == 'latest' else f'tag/{tag}'
 568         self._report_error(
 569             f'Unable to {action}{delim} visit  '
 570             f'https://github.com/{self.requested_repo}/releases/{path}', True)
 571
 572
 573 def run_update(ydl):
 574     """Update the program file with the latest version from the repository
 575     @returns    Whether there was a successful update (No update = False)
 576     """
 577     deprecation_warning(
 578         '"yt_dlp.update.run_update(ydl)" is deprecated and may be removed in a future version. '
 579         'Use "yt_dlp.update.Updater(ydl).update()" instead')
 580     return Updater(ydl).update()
 581
 582
 583 __all__ = ['Updater']