yt_dlp/extractor/boosty.py

   1 import json
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from .youtube import YoutubeIE
   6 from ..utils import (
   7     ExtractorError,
   8     bug_reports_message,
   9     int_or_none,
  10     qualities,
  11     str_or_none,
  12     url_or_none,
  13 )
  14 from ..utils.traversal import traverse_obj
  15
  16
  17 class BoostyIE(InfoExtractor):
  18     _VALID_URL = r'https?://(?:www\.)?boosty\.to/(?P<user>[^/#?]+)/posts/(?P<post_id>[^/#?]+)'
  19     _TESTS = [{
  20         # single ok_video
  21         'url': 'https://boosty.to/kuplinov/posts/e55d050c-e3bb-4873-a7db-ac7a49b40c38',
  22         'info_dict': {
  23             'id': 'd7473824-352e-48e2-ae53-d4aa39459968',
  24             'title': 'phasma_3',
  25             'channel': 'Kuplinov',
  26             'channel_id': '7958701',
  27             'timestamp': 1655031975,
  28             'upload_date': '20220612',
  29             'release_timestamp': 1655049000,
  30             'release_date': '20220612',
  31             'modified_timestamp': 1668680993,
  32             'modified_date': '20221117',
  33             'tags': ['куплинов', 'phasmophobia'],
  34             'like_count': int,
  35             'ext': 'mp4',
  36             'duration': 105,
  37             'view_count': int,
  38             'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
  39         },
  40     }, {
  41         # multiple ok_video
  42         'url': 'https://boosty.to/maddyson/posts/0c652798-3b35-471f-8b48-a76a0b28736f',
  43         'info_dict': {
  44             'id': '0c652798-3b35-471f-8b48-a76a0b28736f',
  45             'title': 'то что не пропустил юта6',
  46             'channel': 'Илья Давыдов',
  47             'channel_id': '6808257',
  48             'timestamp': 1694017040,
  49             'upload_date': '20230906',
  50             'release_timestamp': 1694017040,
  51             'release_date': '20230906',
  52             'modified_timestamp': 1694071178,
  53             'modified_date': '20230907',
  54             'like_count': int,
  55         },
  56         'playlist_count': 3,
  57         'playlist': [{
  58             'info_dict': {
  59                 'id': 'cc325a9f-a563-41c6-bf47-516c1b506c9a',
  60                 'title': 'то что не пропустил юта6',
  61                 'channel': 'Илья Давыдов',
  62                 'channel_id': '6808257',
  63                 'timestamp': 1694017040,
  64                 'upload_date': '20230906',
  65                 'release_timestamp': 1694017040,
  66                 'release_date': '20230906',
  67                 'modified_timestamp': 1694071178,
  68                 'modified_date': '20230907',
  69                 'like_count': int,
  70                 'ext': 'mp4',
  71                 'duration': 31204,
  72                 'view_count': int,
  73                 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
  74             },
  75         }, {
  76             'info_dict': {
  77                 'id': 'd07b0a72-9493-4512-b54e-55ce468fd4b7',
  78                 'title': 'то что не пропустил юта6',
  79                 'channel': 'Илья Давыдов',
  80                 'channel_id': '6808257',
  81                 'timestamp': 1694017040,
  82                 'upload_date': '20230906',
  83                 'release_timestamp': 1694017040,
  84                 'release_date': '20230906',
  85                 'modified_timestamp': 1694071178,
  86                 'modified_date': '20230907',
  87                 'like_count': int,
  88                 'ext': 'mp4',
  89                 'duration': 25704,
  90                 'view_count': int,
  91                 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
  92             },
  93         }, {
  94             'info_dict': {
  95                 'id': '4a3bba32-78c8-422a-9432-2791aff60b42',
  96                 'title': 'то что не пропустил юта6',
  97                 'channel': 'Илья Давыдов',
  98                 'channel_id': '6808257',
  99                 'timestamp': 1694017040,
 100                 'upload_date': '20230906',
 101                 'release_timestamp': 1694017040,
 102                 'release_date': '20230906',
 103                 'modified_timestamp': 1694071178,
 104                 'modified_date': '20230907',
 105                 'like_count': int,
 106                 'ext': 'mp4',
 107                 'duration': 31867,
 108                 'view_count': int,
 109                 'thumbnail': r're:^https://i\.mycdn\.me/videoPreview\?',
 110             },
 111         }],
 112     }, {
 113         # single external video (youtube)
 114         'url': 'https://boosty.to/denischuzhoy/posts/6094a487-bcec-4cf8-a453-43313b463c38',
 115         'info_dict': {
 116             'id': 'EXelTnve5lY',
 117             'title': 'Послание Президента Федеральному Собранию | Класс народа',
 118             'upload_date': '20210425',
 119             'channel': 'Денис Чужой',
 120             'tags': 'count:10',
 121             'like_count': int,
 122             'ext': 'mp4',
 123             'duration': 816,
 124             'view_count': int,
 125             'thumbnail': r're:^https://i\.ytimg\.com/',
 126             'age_limit': 0,
 127             'availability': 'public',
 128             'categories': list,
 129             'channel_follower_count': int,
 130             'channel_id': 'UCCzVNbWZfYpBfyofCCUD_0w',
 131             'channel_is_verified': bool,
 132             'channel_url': r're:^https://www\.youtube\.com/',
 133             'comment_count': int,
 134             'description': str,
 135             'heatmap': 'count:100',
 136             'live_status': str,
 137             'playable_in_embed': bool,
 138             'uploader': str,
 139             'uploader_id': str,
 140             'uploader_url': r're:^https://www\.youtube\.com/',
 141         },
 142     }]
 143
 144     _MP4_TYPES = ('tiny', 'lowest', 'low', 'medium', 'high', 'full_hd', 'quad_hd', 'ultra_hd')
 145
 146     def _extract_formats(self, player_urls, video_id):
 147         formats = []
 148         quality = qualities(self._MP4_TYPES)
 149         for player_url in traverse_obj(player_urls, lambda _, v: url_or_none(v['url'])):
 150             url = player_url['url']
 151             format_type = player_url.get('type')
 152             if format_type in ('hls', 'hls_live', 'live_ondemand_hls', 'live_playback_hls'):
 153                 formats.extend(self._extract_m3u8_formats(url, video_id, m3u8_id='hls', fatal=False))
 154             elif format_type in ('dash', 'dash_live', 'live_playback_dash'):
 155                 formats.extend(self._extract_mpd_formats(url, video_id, mpd_id='dash', fatal=False))
 156             elif format_type in self._MP4_TYPES:
 157                 formats.append({
 158                     'url': url,
 159                     'ext': 'mp4',
 160                     'format_id': format_type,
 161                     'quality': quality(format_type),
 162                 })
 163             else:
 164                 self.report_warning(f'Unknown format type: {format_type!r}')
 165         return formats
 166
 167     def _real_extract(self, url):
 168         user, post_id = self._match_valid_url(url).group('user', 'post_id')
 169
 170         auth_headers = {}
 171         auth_cookie = self._get_cookies('https://boosty.to/').get('auth')
 172         if auth_cookie is not None:
 173             try:
 174                 auth_data = json.loads(urllib.parse.unquote(auth_cookie.value))
 175                 auth_headers['Authorization'] = f'Bearer {auth_data["accessToken"]}'
 176             except (json.JSONDecodeError, KeyError):
 177                 self.report_warning(f'Failed to extract token from auth cookie{bug_reports_message()}')
 178
 179         post = self._download_json(
 180             f'https://api.boosty.to/v1/blog/{user}/post/{post_id}', post_id,
 181             note='Downloading post data', errnote='Unable to download post data', headers=auth_headers)
 182
 183         post_title = post.get('title')
 184         if not post_title:
 185             self.report_warning('Unable to extract post title. Falling back to parsing html page')
 186             webpage = self._download_webpage(url, video_id=post_id)
 187             post_title = self._og_search_title(webpage, default=None) or self._html_extract_title(webpage)
 188
 189         common_metadata = {
 190             'title': post_title,
 191             **traverse_obj(post, {
 192                 'channel': ('user', 'name', {str}),
 193                 'channel_id': ('user', 'id', {str_or_none}),
 194                 'timestamp': ('createdAt', {int_or_none}),
 195                 'release_timestamp': ('publishTime', {int_or_none}),
 196                 'modified_timestamp': ('updatedAt', {int_or_none}),
 197                 'tags': ('tags', ..., 'title', {str}),
 198                 'like_count': ('count', 'likes', {int_or_none}),
 199             }),
 200         }
 201         entries = []
 202         for item in traverse_obj(post, ('data', ..., {dict})):
 203             item_type = item.get('type')
 204             if item_type == 'video' and url_or_none(item.get('url')):
 205                 entries.append(self.url_result(item['url'], YoutubeIE))
 206             elif item_type == 'ok_video':
 207                 video_id = item.get('id') or post_id
 208                 entries.append({
 209                     'id': video_id,
 210                     'formats': self._extract_formats(item.get('playerUrls'), video_id),
 211                     **common_metadata,
 212                     **traverse_obj(item, {
 213                         'title': ('title', {str}),
 214                         'duration': ('duration', {int_or_none}),
 215                         'view_count': ('viewsCounter', {int_or_none}),
 216                         'thumbnail': (('previewUrl', 'defaultPreview'), {url_or_none}),
 217                     }, get_all=False)})
 218
 219         if not entries and not post.get('hasAccess'):
 220             self.raise_login_required('This post requires a subscription', metadata_available=True)
 221         elif not entries:
 222             raise ExtractorError('No videos found', expected=True)
 223         if len(entries) == 1:
 224             return entries[0]
 225         return self.playlist_result(entries, post_id, post_title, **common_metadata)