[cleanup] Misc (#10807)
[yt-dlp.git] / yt_dlp / update.py
blob4cf3bdc320b323236e265f6155ce29e2414363f8
1 from __future__ import annotations
3 import atexit
4 import contextlib
5 import hashlib
6 import json
7 import os
8 import platform
9 import re
10 import subprocess
11 import sys
12 from dataclasses import dataclass
13 from zipimport import zipimporter
15 from .compat import functools # isort: split
16 from .compat import compat_realpath
17 from .networking import Request
18 from .networking.exceptions import HTTPError, network_exceptions
19 from .utils import (
20 NO_DEFAULT,
21 Popen,
22 deprecation_warning,
23 format_field,
24 remove_end,
25 shell_quote,
26 system_identifier,
27 version_tuple,
29 from .version import (
30 CHANNEL,
31 ORIGIN,
32 RELEASE_GIT_HEAD,
33 UPDATE_HINT,
34 VARIANT,
35 __version__,
38 UPDATE_SOURCES = {
39 'stable': 'yt-dlp/yt-dlp',
40 'nightly': 'yt-dlp/yt-dlp-nightly-builds',
41 'master': 'yt-dlp/yt-dlp-master-builds',
43 REPOSITORY = UPDATE_SOURCES['stable']
44 _INVERSE_UPDATE_SOURCES = {value: key for key, value in UPDATE_SOURCES.items()}
46 _VERSION_RE = re.compile(r'(\d+\.)*\d+')
47 _HASH_PATTERN = r'[\da-f]{40}'
48 _COMMIT_RE = re.compile(rf'Generated from: https://(?:[^/?#]+/){{3}}commit/(?P<hash>{_HASH_PATTERN})')
50 API_BASE_URL = 'https://api.github.com/repos'
52 # Backwards compatibility variables for the current channel
53 API_URL = f'{API_BASE_URL}/{REPOSITORY}/releases'
56 @functools.cache
57 def _get_variant_and_executable_path():
58 """@returns (variant, executable_path)"""
59 if getattr(sys, 'frozen', False):
60 path = sys.executable
61 if not hasattr(sys, '_MEIPASS'):
62 return 'py2exe', path
63 elif sys._MEIPASS == os.path.dirname(path):
64 return f'{sys.platform}_dir', path
65 elif sys.platform == 'darwin':
66 machine = '_legacy' if version_tuple(platform.mac_ver()[0]) < (10, 15) else ''
67 else:
68 machine = f'_{platform.machine().lower()}'
69 # Ref: https://en.wikipedia.org/wiki/Uname#Examples
70 if machine[1:] in ('x86', 'x86_64', 'amd64', 'i386', 'i686'):
71 machine = '_x86' if platform.architecture()[0][:2] == '32' else ''
72 # sys.executable returns a /tmp/ path for staticx builds (linux_static)
73 # Ref: https://staticx.readthedocs.io/en/latest/usage.html#run-time-information
74 if static_exe_path := os.getenv('STATICX_PROG_PATH'):
75 path = static_exe_path
76 return f'{remove_end(sys.platform, "32")}{machine}_exe', path
78 path = os.path.dirname(__file__)
79 if isinstance(__loader__, zipimporter):
80 return 'zip', os.path.join(path, '..')
81 elif (os.path.basename(sys.argv[0]) in ('__main__.py', '-m')
82 and os.path.exists(os.path.join(path, '../.git/HEAD'))):
83 return 'source', path
84 return 'unknown', path
87 def detect_variant():
88 return VARIANT or _get_variant_and_executable_path()[0]
91 @functools.cache
92 def current_git_head():
93 if detect_variant() != 'source':
94 return
95 with contextlib.suppress(Exception):
96 stdout, _, _ = Popen.run(
97 ['git', 'rev-parse', '--short', 'HEAD'],
98 text=True, cwd=os.path.dirname(os.path.abspath(__file__)),
99 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
100 if re.fullmatch('[0-9a-f]+', stdout.strip()):
101 return stdout.strip()
104 _FILE_SUFFIXES = {
105 'zip': '',
106 'py2exe': '_min.exe',
107 'win_exe': '.exe',
108 'win_x86_exe': '_x86.exe',
109 'darwin_exe': '_macos',
110 'darwin_legacy_exe': '_macos_legacy',
111 'linux_exe': '_linux',
112 'linux_aarch64_exe': '_linux_aarch64',
113 'linux_armv7l_exe': '_linux_armv7l',
116 _NON_UPDATEABLE_REASONS = {
117 **{variant: None for variant in _FILE_SUFFIXES}, # Updatable
118 **{variant: f'Auto-update is not supported for unpackaged {name} executable; Re-download the latest release'
119 for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS', 'linux_dir': 'Linux'}.items()},
120 'source': 'You cannot update when running from source code; Use git to pull the latest changes',
121 'unknown': 'You installed yt-dlp from a manual build or with a package manager; Use that to update',
122 'other': 'You are using an unofficial build of yt-dlp; Build the executable again',
126 def is_non_updateable():
127 if UPDATE_HINT:
128 return UPDATE_HINT
129 return _NON_UPDATEABLE_REASONS.get(
130 detect_variant(), _NON_UPDATEABLE_REASONS['unknown' if VARIANT else 'other'])
133 def _get_binary_name():
134 return format_field(_FILE_SUFFIXES, detect_variant(), template='yt-dlp%s', ignore=None, default=None)
137 def _get_system_deprecation():
138 MIN_SUPPORTED, MIN_RECOMMENDED = (3, 8), (3, 9)
140 if sys.version_info > MIN_RECOMMENDED:
141 return None
143 major, minor = sys.version_info[:2]
144 PYTHON_MSG = f'Please update to Python {".".join(map(str, MIN_RECOMMENDED))} or above'
146 if sys.version_info < MIN_SUPPORTED:
147 return f'Python version {major}.{minor} is no longer supported! {PYTHON_MSG}'
149 EXE_MSG_TMPL = ('Support for {} has been deprecated. '
150 'See https://github.com/yt-dlp/yt-dlp/{} for details.\n{}')
151 STOP_MSG = 'You may stop receiving updates on this version at any time!'
152 variant = detect_variant()
154 # Temporary until Windows builds use 3.9, which will drop support for Win7 and 2008ServerR2
155 if variant in ('win_exe', 'win_x86_exe', 'py2exe'):
156 platform_name = platform.platform()
157 if any(platform_name.startswith(f'Windows-{name}') for name in ('7', '2008ServerR2')):
158 return EXE_MSG_TMPL.format('Windows 7/Server 2008 R2', 'issues/10086', STOP_MSG)
159 elif variant == 'py2exe':
160 return EXE_MSG_TMPL.format(
161 'py2exe builds (yt-dlp_min.exe)', 'issues/10087',
162 'In a future update you will be migrated to the PyInstaller-bundled executable. '
163 'This will be done automatically; no action is required on your part')
164 return None
166 # Temporary until aarch64/armv7l build flow is bumped to Ubuntu 20.04 and Python 3.9
167 elif variant in ('linux_aarch64_exe', 'linux_armv7l_exe'):
168 libc_ver = version_tuple(os.confstr('CS_GNU_LIBC_VERSION').partition(' ')[2])
169 if libc_ver < (2, 31):
170 return EXE_MSG_TMPL.format('system glibc version < 2.31', 'pull/8638', STOP_MSG)
171 return None
173 return f'Support for Python version {major}.{minor} has been deprecated. {PYTHON_MSG}'
176 def _sha256_file(path):
177 h = hashlib.sha256()
178 mv = memoryview(bytearray(128 * 1024))
179 with open(os.path.realpath(path), 'rb', buffering=0) as f:
180 for n in iter(lambda: f.readinto(mv), 0):
181 h.update(mv[:n])
182 return h.hexdigest()
185 def _make_label(origin, tag, version=None):
186 if '/' in origin:
187 channel = _INVERSE_UPDATE_SOURCES.get(origin, origin)
188 else:
189 channel = origin
190 label = f'{channel}@{tag}'
191 if version and version != tag:
192 label += f' build {version}'
193 if channel != origin:
194 label += f' from {origin}'
195 return label
198 @dataclass
199 class UpdateInfo:
201 Update target information
203 Can be created by `query_update()` or manually.
205 Attributes:
206 tag The release tag that will be updated to. If from query_update,
207 the value is after API resolution and update spec processing.
208 The only property that is required.
209 version The actual numeric version (if available) of the binary to be updated to,
210 after API resolution and update spec processing. (default: None)
211 requested_version Numeric version of the binary being requested (if available),
212 after API resolution only. (default: None)
213 commit Commit hash (if available) of the binary to be updated to,
214 after API resolution and update spec processing. (default: None)
215 This value will only match the RELEASE_GIT_HEAD of prerelease builds.
216 binary_name Filename of the binary to be updated to. (default: current binary name)
217 checksum Expected checksum (if available) of the binary to be
218 updated to. (default: None)
220 tag: str
221 version: str | None = None
222 requested_version: str | None = None
223 commit: str | None = None
225 binary_name: str | None = _get_binary_name() # noqa: RUF009: Always returns the same value
226 checksum: str | None = None
228 _has_update = True
231 class Updater:
232 # XXX: use class variables to simplify testing
233 _channel = CHANNEL
234 _origin = ORIGIN
235 _update_sources = UPDATE_SOURCES
237 def __init__(self, ydl, target: str | None = None):
238 self.ydl = ydl
239 # For backwards compat, target needs to be treated as if it could be None
240 self.requested_channel, sep, self.requested_tag = (target or self._channel).rpartition('@')
241 # Check if requested_tag is actually the requested repo/channel
242 if not sep and ('/' in self.requested_tag or self.requested_tag in self._update_sources):
243 self.requested_channel = self.requested_tag
244 self.requested_tag: str = None # type: ignore (we set it later)
245 elif not self.requested_channel:
246 # User did not specify a channel, so we are requesting the default channel
247 self.requested_channel = self._channel.partition('@')[0]
249 # --update should not be treated as an exact tag request even if CHANNEL has a @tag
250 self._exact = bool(target) and target != self._channel
251 if not self.requested_tag:
252 # User did not specify a tag, so we request 'latest' and track that no exact tag was passed
253 self.requested_tag = 'latest'
254 self._exact = False
256 if '/' in self.requested_channel:
257 # requested_channel is actually a repository
258 self.requested_repo = self.requested_channel
259 if not self.requested_repo.startswith('yt-dlp/') and self.requested_repo != self._origin:
260 self.ydl.report_warning(
261 f'You are switching to an {self.ydl._format_err("unofficial", "red")} executable '
262 f'from {self.ydl._format_err(self.requested_repo, self.ydl.Styles.EMPHASIS)}. '
263 f'Run {self.ydl._format_err("at your own risk", "light red")}')
264 self._block_restart('Automatically restarting into custom builds is disabled for security reasons')
265 else:
266 # Check if requested_channel resolves to a known repository or else raise
267 self.requested_repo = self._update_sources.get(self.requested_channel)
268 if not self.requested_repo:
269 self._report_error(
270 f'Invalid update channel {self.requested_channel!r} requested. '
271 f'Valid channels are {", ".join(self._update_sources)}', True)
273 self._identifier = f'{detect_variant()} {system_identifier()}'
275 @property
276 def current_version(self):
277 """Current version"""
278 return __version__
280 @property
281 def current_commit(self):
282 """Current commit hash"""
283 return RELEASE_GIT_HEAD
285 def _download_asset(self, name, tag=None):
286 if not tag:
287 tag = self.requested_tag
289 path = 'latest/download' if tag == 'latest' else f'download/{tag}'
290 url = f'https://github.com/{self.requested_repo}/releases/{path}/{name}'
291 self.ydl.write_debug(f'Downloading {name} from {url}')
292 return self.ydl.urlopen(url).read()
294 def _call_api(self, tag):
295 tag = f'tags/{tag}' if tag != 'latest' else tag
296 url = f'{API_BASE_URL}/{self.requested_repo}/releases/{tag}'
297 self.ydl.write_debug(f'Fetching release info: {url}')
298 return json.loads(self.ydl.urlopen(Request(url, headers={
299 'Accept': 'application/vnd.github+json',
300 'User-Agent': 'yt-dlp',
301 'X-GitHub-Api-Version': '2022-11-28',
302 })).read().decode())
304 def _get_version_info(self, tag: str) -> tuple[str | None, str | None]:
305 if _VERSION_RE.fullmatch(tag):
306 return tag, None
308 api_info = self._call_api(tag)
310 if tag == 'latest':
311 requested_version = api_info['tag_name']
312 else:
313 match = re.search(rf'\s+(?P<version>{_VERSION_RE.pattern})$', api_info.get('name', ''))
314 requested_version = match.group('version') if match else None
316 if re.fullmatch(_HASH_PATTERN, api_info.get('target_commitish', '')):
317 target_commitish = api_info['target_commitish']
318 else:
319 match = _COMMIT_RE.match(api_info.get('body', ''))
320 target_commitish = match.group('hash') if match else None
322 if not (requested_version or target_commitish):
323 self._report_error('One of either version or commit hash must be available on the release', expected=True)
325 return requested_version, target_commitish
327 def _download_update_spec(self, source_tags):
328 for tag in source_tags:
329 try:
330 return self._download_asset('_update_spec', tag=tag).decode()
331 except network_exceptions as error:
332 if isinstance(error, HTTPError) and error.status == 404:
333 continue
334 self._report_network_error(f'fetch update spec: {error}')
335 return None
337 self._report_error(
338 f'The requested tag {self.requested_tag} does not exist for {self.requested_repo}', True)
339 return None
341 def _process_update_spec(self, lockfile: str, resolved_tag: str):
342 lines = lockfile.splitlines()
343 is_version2 = any(line.startswith('lockV2 ') for line in lines)
345 for line in lines:
346 if is_version2:
347 if not line.startswith(f'lockV2 {self.requested_repo} '):
348 continue
349 _, _, tag, pattern = line.split(' ', 3)
350 else:
351 if not line.startswith('lock '):
352 continue
353 _, tag, pattern = line.split(' ', 2)
355 if re.match(pattern, self._identifier):
356 if _VERSION_RE.fullmatch(tag):
357 if not self._exact:
358 return tag
359 elif self._version_compare(tag, resolved_tag):
360 return resolved_tag
361 elif tag != resolved_tag:
362 continue
364 self._report_error(
365 f'yt-dlp cannot be updated to {resolved_tag} since you are on an older Python version', True)
366 return None
368 return resolved_tag
370 def _version_compare(self, a: str, b: str):
372 Compare two version strings
374 This function SHOULD NOT be called if self._exact == True
376 if _VERSION_RE.fullmatch(f'{a}.{b}'):
377 return version_tuple(a) >= version_tuple(b)
378 return a == b
380 def query_update(self, *, _output=False) -> UpdateInfo | None:
381 """Fetches info about the available update
382 @returns An `UpdateInfo` if there is an update available, else None
384 if not self.requested_repo:
385 self._report_error('No target repository could be determined from input')
386 return None
388 try:
389 requested_version, target_commitish = self._get_version_info(self.requested_tag)
390 except network_exceptions as e:
391 self._report_network_error(f'obtain version info ({e})', delim='; Please try again later or')
392 return None
394 if self._exact and self._origin != self.requested_repo:
395 has_update = True
396 elif requested_version:
397 if self._exact:
398 has_update = self.current_version != requested_version
399 else:
400 has_update = not self._version_compare(self.current_version, requested_version)
401 elif target_commitish:
402 has_update = target_commitish != self.current_commit
403 else:
404 has_update = False
406 resolved_tag = requested_version if self.requested_tag == 'latest' else self.requested_tag
407 current_label = _make_label(self._origin, self._channel.partition('@')[2] or self.current_version, self.current_version)
408 requested_label = _make_label(self.requested_repo, resolved_tag, requested_version)
409 latest_or_requested = f'{"Latest" if self.requested_tag == "latest" else "Requested"} version: {requested_label}'
410 if not has_update:
411 if _output:
412 self.ydl.to_screen(f'{latest_or_requested}\nyt-dlp is up to date ({current_label})')
413 return None
415 update_spec = self._download_update_spec(('latest', None) if requested_version else (None,))
416 if not update_spec:
417 return None
418 # `result_` prefixed vars == post-_process_update_spec() values
419 result_tag = self._process_update_spec(update_spec, resolved_tag)
420 if not result_tag or result_tag == self.current_version:
421 return None
422 elif result_tag == resolved_tag:
423 result_version = requested_version
424 elif _VERSION_RE.fullmatch(result_tag):
425 result_version = result_tag
426 else: # actual version being updated to is unknown
427 result_version = None
429 checksum = None
430 # Non-updateable variants can get update_info but need to skip checksum
431 if not is_non_updateable():
432 try:
433 hashes = self._download_asset('SHA2-256SUMS', result_tag)
434 except network_exceptions as error:
435 if not isinstance(error, HTTPError) or error.status != 404:
436 self._report_network_error(f'fetch checksums: {error}')
437 return None
438 self.ydl.report_warning('No hash information found for the release, skipping verification')
439 else:
440 for ln in hashes.decode().splitlines():
441 if ln.endswith(_get_binary_name()):
442 checksum = ln.split()[0]
443 break
444 if not checksum:
445 self.ydl.report_warning('The hash could not be found in the checksum file, skipping verification')
447 if _output:
448 update_label = _make_label(self.requested_repo, result_tag, result_version)
449 self.ydl.to_screen(
450 f'Current version: {current_label}\n{latest_or_requested}'
451 + (f'\nUpgradable to: {update_label}' if update_label != requested_label else ''))
453 return UpdateInfo(
454 tag=result_tag,
455 version=result_version,
456 requested_version=requested_version,
457 commit=target_commitish if result_tag == resolved_tag else None,
458 checksum=checksum)
460 def update(self, update_info=NO_DEFAULT):
461 """Update yt-dlp executable to the latest version
462 @param update_info `UpdateInfo | None` as returned by query_update()
464 if update_info is NO_DEFAULT:
465 update_info = self.query_update(_output=True)
466 if not update_info:
467 return False
469 err = is_non_updateable()
470 if err:
471 self._report_error(err, True)
472 return False
474 self.ydl.to_screen(f'Current Build Hash: {_sha256_file(self.filename)}')
476 update_label = _make_label(self.requested_repo, update_info.tag, update_info.version)
477 self.ydl.to_screen(f'Updating to {update_label} ...')
479 directory = os.path.dirname(self.filename)
480 if not os.access(self.filename, os.W_OK):
481 return self._report_permission_error(self.filename)
482 elif not os.access(directory, os.W_OK):
483 return self._report_permission_error(directory)
485 new_filename, old_filename = f'{self.filename}.new', f'{self.filename}.old'
486 if detect_variant() == 'zip': # Can be replaced in-place
487 new_filename, old_filename = self.filename, None
489 try:
490 if os.path.exists(old_filename or ''):
491 os.remove(old_filename)
492 except OSError:
493 return self._report_error('Unable to remove the old version')
495 try:
496 newcontent = self._download_asset(update_info.binary_name, update_info.tag)
497 except network_exceptions as e:
498 if isinstance(e, HTTPError) and e.status == 404:
499 return self._report_error(
500 f'The requested tag {self.requested_repo}@{update_info.tag} does not exist', True)
501 return self._report_network_error(f'fetch updates: {e}', tag=update_info.tag)
503 if not update_info.checksum:
504 self._block_restart('Automatically restarting into unverified builds is disabled for security reasons')
505 elif hashlib.sha256(newcontent).hexdigest() != update_info.checksum:
506 return self._report_network_error('verify the new executable', tag=update_info.tag)
508 try:
509 with open(new_filename, 'wb') as outf:
510 outf.write(newcontent)
511 except OSError:
512 return self._report_permission_error(new_filename)
514 if old_filename:
515 mask = os.stat(self.filename).st_mode
516 try:
517 os.rename(self.filename, old_filename)
518 except OSError:
519 return self._report_error('Unable to move current version')
521 try:
522 os.rename(new_filename, self.filename)
523 except OSError:
524 self._report_error('Unable to overwrite current version')
525 return os.rename(old_filename, self.filename)
527 variant = detect_variant()
528 if variant.startswith('win') or variant == 'py2exe':
529 atexit.register(Popen, f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{old_filename}"',
530 shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
531 elif old_filename:
532 try:
533 os.remove(old_filename)
534 except OSError:
535 self._report_error('Unable to remove the old version')
537 try:
538 os.chmod(self.filename, mask)
539 except OSError:
540 return self._report_error(
541 f'Unable to set permissions. Run: sudo chmod a+rx {shell_quote(self.filename)}')
543 self.ydl.to_screen(f'Updated yt-dlp to {update_label}')
544 return True
546 @functools.cached_property
547 def filename(self):
548 """Filename of the executable"""
549 return compat_realpath(_get_variant_and_executable_path()[1])
551 @functools.cached_property
552 def cmd(self):
553 """The command-line to run the executable, if known"""
554 # There is no sys.orig_argv in py < 3.10. Also, it can be [] when frozen
555 if getattr(sys, 'orig_argv', None):
556 return sys.orig_argv
557 elif getattr(sys, 'frozen', False):
558 return sys.argv
560 def restart(self):
561 """Restart the executable"""
562 assert self.cmd, 'Must be frozen or Py >= 3.10'
563 self.ydl.write_debug(f'Restarting: {shell_quote(self.cmd)}')
564 _, _, returncode = Popen.run(self.cmd)
565 return returncode
567 def _block_restart(self, msg):
568 def wrapper():
569 self._report_error(f'{msg}. Restart yt-dlp to use the updated version', expected=True)
570 return self.ydl._download_retcode
571 self.restart = wrapper
573 def _report_error(self, msg, expected=False):
574 self.ydl.report_error(msg, tb=False if expected else None)
575 self.ydl._download_retcode = 100
577 def _report_permission_error(self, file):
578 self._report_error(f'Unable to write to {file}; try running as administrator', True)
580 def _report_network_error(self, action, delim=';', tag=None):
581 if not tag:
582 tag = self.requested_tag
583 path = tag if tag == 'latest' else f'tag/{tag}'
584 self._report_error(
585 f'Unable to {action}{delim} visit '
586 f'https://github.com/{self.requested_repo}/releases/{path}', True)
588 # XXX: Everything below this line in this class is deprecated / for compat only
589 @property
590 def _target_tag(self):
591 """Deprecated; requested tag with 'tags/' prepended when necessary for API calls"""
592 return f'tags/{self.requested_tag}' if self.requested_tag != 'latest' else self.requested_tag
594 def _check_update(self):
595 """Deprecated; report whether there is an update available"""
596 return bool(self.query_update(_output=True))
598 def __getattr__(self, attribute: str):
599 """Compat getter function for deprecated attributes"""
600 deprecated_props_map = {
601 'check_update': '_check_update',
602 'target_tag': '_target_tag',
603 'target_channel': 'requested_channel',
605 update_info_props_map = {
606 'has_update': '_has_update',
607 'new_version': 'version',
608 'latest_version': 'requested_version',
609 'release_name': 'binary_name',
610 'release_hash': 'checksum',
613 if attribute not in deprecated_props_map and attribute not in update_info_props_map:
614 raise AttributeError(f'{type(self).__name__!r} object has no attribute {attribute!r}')
616 msg = f'{type(self).__name__}.{attribute} is deprecated and will be removed in a future version'
617 if attribute in deprecated_props_map:
618 source_name = deprecated_props_map[attribute]
619 if not source_name.startswith('_'):
620 msg += f'. Please use {source_name!r} instead'
621 source = self
622 mapping = deprecated_props_map
624 else: # attribute in update_info_props_map
625 msg += '. Please call query_update() instead'
626 source = self.query_update()
627 if source is None:
628 source = UpdateInfo('', None, None, None)
629 source._has_update = False
630 mapping = update_info_props_map
632 deprecation_warning(msg)
633 for target_name, source_name in mapping.items():
634 value = getattr(source, source_name)
635 setattr(self, target_name, value)
637 return getattr(self, attribute)
640 def run_update(ydl):
641 """Update the program file with the latest version from the repository
642 @returns Whether there was a successful update (No update = False)
644 return Updater(ydl).update()
647 __all__ = ['Updater']