yt_dlp/update.py

   1 from __future__ import annotations
   2
   3 import atexit
   4 import contextlib
   5 import functools
   6 import hashlib
   7 import json
   8 import os
   9 import platform
  10 import re
  11 import subprocess
  12 import sys
  13 from dataclasses import dataclass
  14 from zipimport import zipimporter
  15
  16 from .networking import Request
  17 from .networking.exceptions import HTTPError, network_exceptions
  18 from .utils import (
  19     NO_DEFAULT,
  20     Popen,
  21     deprecation_warning,
  22     format_field,
  23     remove_end,
  24     shell_quote,
  25     system_identifier,
  26     version_tuple,
  27 )
  28 from .version import (
  29     CHANNEL,
  30     ORIGIN,
  31     RELEASE_GIT_HEAD,
  32     UPDATE_HINT,
  33     VARIANT,
  34     __version__,
  35 )
  36
  37 UPDATE_SOURCES = {
  38     'stable': 'yt-dlp/yt-dlp',
  39     'nightly': 'yt-dlp/yt-dlp-nightly-builds',
  40     'master': 'yt-dlp/yt-dlp-master-builds',
  41 }
  42 REPOSITORY = UPDATE_SOURCES['stable']
  43 _INVERSE_UPDATE_SOURCES = {value: key for key, value in UPDATE_SOURCES.items()}
  44
  45 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
  46 _HASH_PATTERN = r'[\da-f]{40}'
  47 _COMMIT_RE = re.compile(rf'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
  48
  49 API_BASE_URL = 'https://api.github.com/repos'
  50
  51 # Backwards compatibility variables for the current channel
  52 API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
  53
  54
  55 @functools.cache
  56 def _get_variant_and_executable_path():
  57     """@returns (variant, executable_path)"""
  58     if getattr(sys, 'frozen', False):
  59         path = sys.executable
  60         if not hasattr(sys, '_MEIPASS'):
  61             return 'py2exe', path
  62         elif sys._MEIPASS == os.path.dirname(path):
  63             return f'{sys.platform}_dir', path
  64         elif sys.platform == 'darwin':
  65             machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else ''
  66         else:
  67             machine = f'_{platform.machine().lower()}'
  68             # Ref: https://en.wikipedia.org/wiki/Uname#Examples
  69             if machine[1:] in ('x86', 'x86_64', 'amd64', 'i386', 'i686'):
  70                 machine = '_x86' if platform.architecture()[0][:2] == '32' else ''
  71             # sys.executable returns a /tmp/ path for staticx builds (linux_static)
  72             # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information
  73             if static_exe_path := os.getenv('STATICX_PROG_PATH'):
  74                 path = static_exe_path
  75         return f'{remove_end(sys.platform, "32")}{machine}_exe', path
  76
  77     path = os.path.dirname(__file__)
  78     if isinstance(__loader__, zipimporter):
  79         return 'zip', os.path.join(path, '..')
  80     elif (os.path.basename(sys.argv[0]) in ('__main__.py', '-m')
  81           and os.path.exists(os.path.join(path, '../.git/HEAD'))):
  82         return 'source', path
  83     return 'unknown', path
  84
  85
  86 def detect_variant():
  87     return VARIANT or _get_variant_and_executable_path()[0]
  88
  89
  90 @functools.cache
  91 def current_git_head():
  92     if detect_variant() != 'source':
  93         return
  94     with contextlib.suppress(Exception):
  95         stdout, _, _ = Popen.run(
  96             ['git', 'rev-parse', '--short', 'HEAD'],
  97             text=True, cwd=os.path.dirname(os.path.abspath(__file__)),
  98             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  99         if re.fullmatch('[0-9a-f]+', stdout.strip()):
 100             return stdout.strip()
 101
 102
 103 _FILE_SUFFIXES = {
 104     'zip': '',
 105     'win_exe': '.exe',
 106     'win_x86_exe': '_x86.exe',
 107     'darwin_exe': '_macos',
 108     'darwin_legacy_exe': '_macos_legacy',
 109     'linux_exe': '_linux',
 110     'linux_aarch64_exe': '_linux_aarch64',
 111     'linux_armv7l_exe': '_linux_armv7l',
 112 }
 113
 114 _NON_UPDATEABLE_REASONS = {
 115     **{variant: None for variant in _FILE_SUFFIXES},  # Updatable
 116     **{variant: f'Auto-update is not supported for unpackaged {name} executable; Re-download the latest release'
 117        for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS', 'linux_dir': 'Linux'}.items()},
 118     'py2exe': 'py2exe is no longer supported by yt-dlp; This executable cannot be updated',
 119     'source': 'You cannot update when running from source code; Use git to pull the latest changes',
 120     'unknown': 'You installed yt-dlp from a manual build or with a package manager; Use that to update',
 121     'other': 'You are using an unofficial build of yt-dlp; Build the executable again',
 122 }
 123
 124
 125 def is_non_updateable():
 126     if UPDATE_HINT:
 127         return UPDATE_HINT
 128     return _NON_UPDATEABLE_REASONS.get(
 129         detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
 130
 131
 132 def _get_binary_name():
 133     return format_field(_FILE_SUFFIXES, detect_variant(), template='yt-dlp%s', ignore=None, default=None)
 134
 135
 136 def _get_system_deprecation():
 137     MIN_SUPPORTED, MIN_RECOMMENDED = (3, 9), (3, 9)
 138
 139     if sys.version_info > MIN_RECOMMENDED:
 140         return None
 141
 142     major, minor = sys.version_info[:2]
 143     PYTHON_MSG = f'Please update to Python {".".join(map(str, MIN_RECOMMENDED))} or above'
 144
 145     if sys.version_info < MIN_SUPPORTED:
 146         return f'Python version {major}.{minor} is no longer supported! {PYTHON_MSG}'
 147
 148     return f'Support for Python version {major}.{minor} has been deprecated. {PYTHON_MSG}'
 149
 150
 151 def _sha256_file(path):
 152     h = hashlib.sha256()
 153     mv = memoryview(bytearray(128 * 1024))
 154     with open(os.path.realpath(path), 'rb', buffering=0) as f:
 155         for n in iter(lambda: f.readinto(mv), 0):
 156             h.update(mv[:n])
 157     return h.hexdigest()
 158
 159
 160 def _make_label(origin, tag, version=None):
 161     if '/' in origin:
 162         channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
 163     else:
 164         channel = origin
 165     label = f'{channel}@{tag}'
 166     if version and version != tag:
 167         label += f' build {version}'
 168     if channel != origin:
 169         label += f' from {origin}'
 170     return label
 171
 172
 173 @dataclass
 174 class UpdateInfo:
 175     """
 176     Update target information
 177
 178     Can be created by `query_update()` or manually.
 179
 180     Attributes:
 181         tag                 The release tag that will be updated to. If from query_update,
 182                             the value is after API resolution and update spec processing.
 183                             The only property that is required.
 184         version             The actual numeric version (if available) of the binary to be updated to,
 185                             after API resolution and update spec processing. (default: None)
 186         requested_version   Numeric version of the binary being requested (if available),
 187                             after API resolution only. (default: None)
 188         commit              Commit hash (if available) of the binary to be updated to,
 189                             after API resolution and update spec processing. (default: None)
 190                             This value will only match the RELEASE_GIT_HEAD of prerelease builds.
 191         binary_name         Filename of the binary to be updated to. (default: current binary name)
 192         checksum            Expected checksum (if available) of the binary to be
 193                             updated to. (default: None)
 194     """
 195     tag: str
 196     version: str | None = None
 197     requested_version: str | None = None
 198     commit: str | None = None
 199
 200     binary_name: str | None = _get_binary_name()  # noqa: RUF009: Always returns the same value
 201     checksum: str | None = None
 202
 203
 204 class Updater:
 205     # XXX: use class variables to simplify testing
 206     _channel = CHANNEL
 207     _origin = ORIGIN
 208     _update_sources = UPDATE_SOURCES
 209
 210     def __init__(self, ydl, target: str | None = None):
 211         self.ydl = ydl
 212         # For backwards compat, target needs to be treated as if it could be None
 213         self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
 214         # Check if requested_tag is actually the requested repo/channel
 215         if not sep and ('/' in self.requested_tag or self.requested_tag in self._update_sources):
 216             self.requested_channel = self.requested_tag
 217             self.requested_tag: str = None  # type: ignore (we set it later)
 218         elif not self.requested_channel:
 219             # User did not specify a channel, so we are requesting the default channel
 220             self.requested_channel = self._channel.partition('@')[0]
 221
 222         # --update should not be treated as an exact tag request even if CHANNEL has a @tag
 223         self._exact = bool(target) and target != self._channel
 224         if not self.requested_tag:
 225             # User did not specify a tag, so we request 'latest' and track that no exact tag was passed
 226             self.requested_tag = 'latest'
 227             self._exact = False
 228
 229         if '/' in self.requested_channel:
 230             # requested_channel is actually a repository
 231             self.requested_repo = self.requested_channel
 232             if not self.requested_repo.startswith('yt-dlp/') and self.requested_repo != self._origin:
 233                 self.ydl.report_warning(
 234                     f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
 235                     f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}. '
 236                     f'Run {self.ydl._format_err("at your own risk", "light red")}')
 237                 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
 238         else:
 239             # Check if requested_channel resolves to a known repository or else raise
 240             self.requested_repo = self._update_sources.get(self.requested_channel)
 241             if not self.requested_repo:
 242                 self._report_error(
 243                     f'Invalid update channel {self.requested_channel!r} requested. '
 244                     f'Valid channels are {", ".join(self._update_sources)}', True)
 245
 246         self._identifier = f'{detect_variant()} {system_identifier()}'
 247
 248     @property
 249     def current_version(self):
 250         """Current version"""
 251         return __version__
 252
 253     @property
 254     def current_commit(self):
 255         """Current commit hash"""
 256         return RELEASE_GIT_HEAD
 257
 258     def _download_asset(self, name, tag=None):
 259         if not tag:
 260             tag = self.requested_tag
 261
 262         path = 'latest/download' if tag == 'latest' else f'download/{tag}'
 263         url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
 264         self.ydl.write_debug(f'Downloading {name} from {url}')
 265         return self.ydl.urlopen(url).read()
 266
 267     def _call_api(self, tag):
 268         tag = f'tags/{tag}' if tag != 'latest' else tag
 269         url = f'{API_BASE_URL}/{self.requested_repo}/releases/{tag}'
 270         self.ydl.write_debug(f'Fetching release info: {url}')
 271         return json.loads(self.ydl.urlopen(Request(url, headers={
 272             'Accept': 'application/vnd.github+json',
 273             'User-Agent': 'yt-dlp',
 274             'X-GitHub-Api-Version': '2022-11-28',
 275         })).read().decode())
 276
 277     def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
 278         if _VERSION_RE.fullmatch(tag):
 279             return tag, None
 280
 281         api_info = self._call_api(tag)
 282
 283         if tag == 'latest':
 284             requested_version = api_info['tag_name']
 285         else:
 286             match = re.search(rf'\s+(?P<version>{_VERSION_RE.pattern})$', api_info.get('name', ''))
 287             requested_version = match.group('version') if match else None
 288
 289         if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish', '')):
 290             target_commitish = api_info['target_commitish']
 291         else:
 292             match = _COMMIT_RE.match(api_info.get('body', ''))
 293             target_commitish = match.group('hash') if match else None
 294
 295         if not (requested_version or target_commitish):
 296             self._report_error('One of either version or commit hash must be available on the release', expected=True)
 297
 298         return requested_version, target_commitish
 299
 300     def _download_update_spec(self, source_tags):
 301         for tag in source_tags:
 302             try:
 303                 return self._download_asset('_update_spec', tag=tag).decode()
 304             except network_exceptions as error:
 305                 if isinstance(error, HTTPError) and error.status == 404:
 306                     continue
 307                 self._report_network_error(f'fetch update spec: {error}')
 308                 return None
 309
 310         self._report_error(
 311             f'The requested tag {self.requested_tag} does not exist for {self.requested_repo}', True)
 312         return None
 313
 314     def _process_update_spec(self, lockfile: str, resolved_tag: str):
 315         lines = lockfile.splitlines()
 316         is_version2 = any(line.startswith('lockV2 ') for line in lines)
 317
 318         for line in lines:
 319             if is_version2:
 320                 if not line.startswith(f'lockV2 {self.requested_repo} '):
 321                     continue
 322                 _, _, tag, pattern = line.split(' ', 3)
 323             else:
 324                 if not line.startswith('lock '):
 325                     continue
 326                 _, tag, pattern = line.split(' ', 2)
 327
 328             if re.match(pattern, self._identifier):
 329                 if _VERSION_RE.fullmatch(tag):
 330                     if not self._exact:
 331                         return tag
 332                     elif self._version_compare(tag, resolved_tag):
 333                         return resolved_tag
 334                 elif tag != resolved_tag:
 335                     continue
 336
 337                 self._report_error(
 338                     f'yt-dlp cannot be updated to {resolved_tag} since you are on an older Python version '
 339                     'or your operating system is not compatible with the requested build', True)
 340                 return None
 341
 342         return resolved_tag
 343
 344     def _version_compare(self, a: str, b: str):
 345         """
 346         Compare two version strings
 347
 348         This function SHOULD NOT be called if self._exact == True
 349         """
 350         if _VERSION_RE.fullmatch(f'{a}.{b}'):
 351             return version_tuple(a) >= version_tuple(b)
 352         return a == b
 353
 354     def query_update(self, *, _output=False) -> UpdateInfo | None:
 355         """Fetches info about the available update
 356         @returns   An `UpdateInfo` if there is an update available, else None
 357         """
 358         if not self.requested_repo:
 359             self._report_error('No target repository could be determined from input')
 360             return None
 361
 362         try:
 363             requested_version, target_commitish = self._get_version_info(self.requested_tag)
 364         except network_exceptions as e:
 365             self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
 366             return None
 367
 368         if self._exact and self._origin != self.requested_repo:
 369             has_update = True
 370         elif requested_version:
 371             if self._exact:
 372                 has_update = self.current_version != requested_version
 373             else:
 374                 has_update = not self._version_compare(self.current_version, requested_version)
 375         elif target_commitish:
 376             has_update = target_commitish != self.current_commit
 377         else:
 378             has_update = False
 379
 380         resolved_tag = requested_version if self.requested_tag == 'latest' else self.requested_tag
 381         current_label = _make_label(self._origin, self._channel.partition('@')[2] or self.current_version, self.current_version)
 382         requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
 383         latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version: {requested_label}'
 384         if not has_update:
 385             if _output:
 386                 self.ydl.to_screen(f'{latest_or_requested}\nyt-dlp is up to date ({current_label})')
 387             return None
 388
 389         update_spec = self._download_update_spec(('latest', None) if requested_version else (None,))
 390         if not update_spec:
 391             return None
 392         # `result_` prefixed vars == post-_process_update_spec() values
 393         result_tag = self._process_update_spec(update_spec, resolved_tag)
 394         if not result_tag or result_tag == self.current_version:
 395             return None
 396         elif result_tag == resolved_tag:
 397             result_version = requested_version
 398         elif _VERSION_RE.fullmatch(result_tag):
 399             result_version = result_tag
 400         else:  # actual version being updated to is unknown
 401             result_version = None
 402
 403         checksum = None
 404         # Non-updateable variants can get update_info but need to skip checksum
 405         if not is_non_updateable():
 406             try:
 407                 hashes = self._download_asset('SHA2-256SUMS', result_tag)
 408             except network_exceptions as error:
 409                 if not isinstance(error, HTTPError) or error.status != 404:
 410                     self._report_network_error(f'fetch checksums: {error}')
 411                     return None
 412                 self.ydl.report_warning('No hash information found for the release, skipping verification')
 413             else:
 414                 for ln in hashes.decode().splitlines():
 415                     if ln.endswith(_get_binary_name()):
 416                         checksum = ln.split()[0]
 417                         break
 418                 if not checksum:
 419                     self.ydl.report_warning('The hash could not be found in the checksum file, skipping verification')
 420
 421         if _output:
 422             update_label = _make_label(self.requested_repo, result_tag, result_version)
 423             self.ydl.to_screen(
 424                 f'Current version: {current_label}\n{latest_or_requested}'
 425                 + (f'\nUpgradable to: {update_label}' if update_label != requested_label else ''))
 426
 427         return UpdateInfo(
 428             tag=result_tag,
 429             version=result_version,
 430             requested_version=requested_version,
 431             commit=target_commitish if result_tag == resolved_tag else None,
 432             checksum=checksum)
 433
 434     def update(self, update_info=NO_DEFAULT):
 435         """Update yt-dlp executable to the latest version
 436         @param update_info  `UpdateInfo | None` as returned by query_update()
 437         """
 438         if update_info is NO_DEFAULT:
 439             update_info = self.query_update(_output=True)
 440         if not update_info:
 441             return False
 442
 443         err = is_non_updateable()
 444         if err:
 445             self._report_error(err, True)
 446             return False
 447
 448         self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
 449
 450         update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
 451         self.ydl.to_screen(f'Updating to {update_label} ...')
 452
 453         directory = os.path.dirname(self.filename)
 454         if not os.access(self.filename, os.W_OK):
 455             return self._report_permission_error(self.filename)
 456         elif not os.access(directory, os.W_OK):
 457             return self._report_permission_error(directory)
 458
 459         new_filename, old_filename = f'{self.filename}.new', f'{self.filename}.old'
 460         if detect_variant() == 'zip':  # Can be replaced in-place
 461             new_filename, old_filename = self.filename, None
 462
 463         try:
 464             if os.path.exists(old_filename or ''):
 465                 os.remove(old_filename)
 466         except OSError:
 467             return self._report_error('Unable to remove the old version')
 468
 469         try:
 470             newcontent = self._download_asset(update_info.binary_name, update_info.tag)
 471         except network_exceptions as e:
 472             if isinstance(e, HTTPError) and e.status == 404:
 473                 return self._report_error(
 474                     f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
 475             return self._report_network_error(f'fetch updates: {e}', tag=update_info.tag)
 476
 477         if not update_info.checksum:
 478             self._block_restart('Automatically restarting into unverified builds is disabled for security reasons')
 479         elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
 480             return self._report_network_error('verify the new executable', tag=update_info.tag)
 481
 482         try:
 483             with open(new_filename, 'wb') as outf:
 484                 outf.write(newcontent)
 485         except OSError:
 486             return self._report_permission_error(new_filename)
 487
 488         if old_filename:
 489             mask = os.stat(self.filename).st_mode
 490             try:
 491                 os.rename(self.filename, old_filename)
 492             except OSError:
 493                 return self._report_error('Unable to move current version')
 494
 495             try:
 496                 os.rename(new_filename, self.filename)
 497             except OSError:
 498                 self._report_error('Unable to overwrite current version')
 499                 return os.rename(old_filename, self.filename)
 500
 501         variant = detect_variant()
 502         if variant.startswith('win'):
 503             atexit.register(Popen, f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{old_filename}"',
 504                             shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 505         elif old_filename:
 506             try:
 507                 os.remove(old_filename)
 508             except OSError:
 509                 self._report_error('Unable to remove the old version')
 510
 511             try:
 512                 os.chmod(self.filename, mask)
 513             except OSError:
 514                 return self._report_error(
 515                     f'Unable to set permissions. Run: sudo chmod a+rx {shell_quote(self.filename)}')
 516
 517         self.ydl.to_screen(f'Updated yt-dlp to {update_label}')
 518         return True
 519
 520     @functools.cached_property
 521     def filename(self):
 522         """Filename of the executable"""
 523         return os.path.realpath(_get_variant_and_executable_path()[1])
 524
 525     @functools.cached_property
 526     def cmd(self):
 527         """The command-line to run the executable, if known"""
 528         # There is no sys.orig_argv in py < 3.10. Also, it can be [] when frozen
 529         if getattr(sys, 'orig_argv', None):
 530             return sys.orig_argv
 531         elif getattr(sys, 'frozen', False):
 532             return sys.argv
 533
 534     def restart(self):
 535         """Restart the executable"""
 536         assert self.cmd, 'Must be frozen or Py >= 3.10'
 537         self.ydl.write_debug(f'Restarting: {shell_quote(self.cmd)}')
 538         _, _, returncode = Popen.run(self.cmd)
 539         return returncode
 540
 541     def _block_restart(self, msg):
 542         def wrapper():
 543             self._report_error(f'{msg}. Restart yt-dlp to use the updated version', expected=True)
 544             return self.ydl._download_retcode
 545         self.restart = wrapper
 546
 547     def _report_error(self, msg, expected=False):
 548         self.ydl.report_error(msg, tb=False if expected else None)
 549         self.ydl._download_retcode = 100
 550
 551     def _report_permission_error(self, file):
 552         self._report_error(f'Unable to write to {file}; try running as administrator', True)
 553
 554     def _report_network_error(self, action, delim=';', tag=None):
 555         if not tag:
 556             tag = self.requested_tag
 557         path = tag if tag == 'latest' else f'tag/{tag}'
 558         self._report_error(
 559             f'Unable to {action}{delim} visit  '
 560             f'https://github.com/{self.requested_repo}/releases/{path}', True)
 561
 562
 563 def run_update(ydl):
 564     """Update the program file with the latest version from the repository
 565     @returns    Whether there was a successful update (No update = False)
 566     """
 567     deprecation_warning(
 568         '"yt_dlp.update.run_update(ydl)" is deprecated and may be removed in a future version. '
 569         'Use "yt_dlp.update.Updater(ydl).update()" instead')
 570     return Updater(ydl).update()
 571
 572
 573 __all__ = ['Updater']