yt_dlp/extractor/trovo.py

   1 import itertools
   2 import json
   3 import random
   4 import string
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     ExtractorError,
   9     format_field,
  10     int_or_none,
  11     str_or_none,
  12     traverse_obj,
  13     try_get,
  14 )
  15
  16
  17 class TrovoBaseIE(InfoExtractor):
  18     _VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
  19     _HEADERS = {'Origin': 'https://trovo.live'}
  20
  21     def _call_api(self, video_id, data):
  22         if 'persistedQuery' in data.get('extensions', {}):
  23             url = 'https://gql.trovo.live'
  24         else:
  25             url = 'https://api-web.trovo.live/graphql'
  26
  27         resp = self._download_json(
  28             url, video_id, data=json.dumps([data]).encode(), headers={'Accept': 'application/json'},
  29             query={
  30                 'qid': ''.join(random.choices(string.ascii_uppercase + string.digits, k=16)),
  31             })[0]
  32         if 'errors' in resp:
  33             raise ExtractorError(f'Trovo said: {resp["errors"][0]["message"]}')
  34         return resp['data'][data['operationName']]
  35
  36     def _extract_streamer_info(self, data):
  37         streamer_info = data.get('streamerInfo') or {}
  38         username = streamer_info.get('userName')
  39         return {
  40             'uploader': streamer_info.get('nickName'),
  41             'uploader_id': str_or_none(streamer_info.get('uid')),
  42             'uploader_url': format_field(username, None, 'https://trovo.live/%s'),
  43         }
  44
  45
  46 class TrovoIE(TrovoBaseIE):
  47     _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:s/)?(?!(?:clip|video)/)(?P<id>(?!s/)[^/?&#]+(?![^#]+[?&]vid=))'
  48     _TESTS = [{
  49         'url': 'https://trovo.live/Exsl',
  50         'only_matching': True,
  51     }, {
  52         'url': 'https://trovo.live/s/SkenonSLive/549759191497',
  53         'only_matching': True,
  54     }, {
  55         'url': 'https://trovo.live/s/zijo987/208251706',
  56         'info_dict': {
  57             'id': '104125853_104125853_1656439572',
  58             'ext': 'flv',
  59             'uploader_url': 'https://trovo.live/zijo987',
  60             'uploader_id': '104125853',
  61             'thumbnail': 'https://livecover.trovo.live/screenshot/73846_104125853_104125853-2022-06-29-04-00-22-852x480.jpg',
  62             'uploader': 'zijo987',
  63             'title': '💥IGRAMO IGRICE UPADAJTE💥2500/5000 2022-06-28 22:01',
  64             'live_status': 'is_live',
  65         },
  66         'skip': 'May not be live',
  67     }]
  68
  69     def _real_extract(self, url):
  70         username = self._match_id(url)
  71         live_info = self._call_api(username, data={
  72             'operationName': 'live_LiveReaderService_GetLiveInfo',
  73             'variables': {
  74                 'params': {
  75                     'userName': username,
  76                 },
  77             },
  78         })
  79         if live_info.get('isLive') == 0:
  80             raise ExtractorError(f'{username} is offline', expected=True)
  81         program_info = live_info['programInfo']
  82         program_id = program_info['id']
  83         title = program_info['title']
  84
  85         formats = []
  86         for stream_info in (program_info.get('streamInfo') or []):
  87             play_url = stream_info.get('playUrl')
  88             if not play_url:
  89                 continue
  90             format_id = stream_info.get('desc')
  91             formats.append({
  92                 'format_id': format_id,
  93                 'height': int_or_none(format_id[:-1]) if format_id else None,
  94                 'url': play_url,
  95                 'tbr': stream_info.get('bitrate'),
  96                 'http_headers': self._HEADERS,
  97             })
  98
  99         info = {
 100             'id': program_id,
 101             'title': title,
 102             'formats': formats,
 103             'thumbnail': program_info.get('coverUrl'),
 104             'is_live': True,
 105         }
 106         info.update(self._extract_streamer_info(live_info))
 107         return info
 108
 109
 110 class TrovoVodIE(TrovoBaseIE):
 111     _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video|s)/(?:[^/]+/\d+[^#]*[?&]vid=)?(?P<id>(?<!/s/)[^/?&#]+)'
 112     _TESTS = [{
 113         'url': 'https://trovo.live/clip/lc-5285890818705062210?ltab=videos',
 114         'params': {'getcomments': True},
 115         'info_dict': {
 116             'id': 'lc-5285890818705062210',
 117             'ext': 'mp4',
 118             'title': 'fatal moaning for a super good🤣🤣',
 119             'uploader': 'OneTappedYou',
 120             'timestamp': 1621628019,
 121             'upload_date': '20210521',
 122             'uploader_id': '100719456',
 123             'duration': 31,
 124             'view_count': int,
 125             'like_count': int,
 126             'comment_count': int,
 127             'comments': 'mincount:1',
 128             'categories': ['Call of Duty: Mobile'],
 129             'uploader_url': 'https://trovo.live/OneTappedYou',
 130             'thumbnail': r're:^https?://.*\.jpg',
 131         },
 132     }, {
 133         'url': 'https://trovo.live/s/SkenonSLive/549759191497?vid=ltv-100829718_100829718_387702301737980280',
 134         'info_dict': {
 135             'id': 'ltv-100829718_100829718_387702301737980280',
 136             'ext': 'mp4',
 137             'timestamp': 1654909624,
 138             'thumbnail': 'http://vod.trovo.live/1f09baf0vodtransger1301120758/ef9ea3f0387702301737980280/coverBySnapshot/coverBySnapshot_10_0.jpg',
 139             'uploader_id': '100829718',
 140             'uploader': 'SkenonSLive',
 141             'title': 'Trovo u secanju, uz par modova i muzike :)',
 142             'uploader_url': 'https://trovo.live/SkenonSLive',
 143             'duration': 10830,
 144             'view_count': int,
 145             'like_count': int,
 146             'upload_date': '20220611',
 147             'comment_count': int,
 148             'categories': ['Minecraft'],
 149         },
 150         'skip': 'Not available',
 151     }, {
 152         'url': 'https://trovo.live/s/Trovo/549756886599?vid=ltv-100264059_100264059_387702304241698583',
 153         'info_dict': {
 154             'id': 'ltv-100264059_100264059_387702304241698583',
 155             'ext': 'mp4',
 156             'timestamp': 1661479563,
 157             'thumbnail': 'http://vod.trovo.live/be5ae591vodtransusw1301120758/cccb9915387702304241698583/coverBySnapshot/coverBySnapshot_10_0.jpg',
 158             'uploader_id': '100264059',
 159             'uploader': 'Trovo',
 160             'title': 'Dev Corner 8/25',
 161             'uploader_url': 'https://trovo.live/Trovo',
 162             'duration': 3753,
 163             'view_count': int,
 164             'like_count': int,
 165             'upload_date': '20220826',
 166             'comment_count': int,
 167             'categories': ['Talk Shows'],
 168         },
 169     }, {
 170         'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
 171         'only_matching': True,
 172     }, {
 173         'url': 'https://trovo.live/s/SkenonSLive/549759191497?foo=bar&vid=ltv-100829718_100829718_387702301737980280',
 174         'only_matching': True,
 175     }]
 176
 177     def _real_extract(self, url):
 178         vid = self._match_id(url)
 179
 180         # NOTE: It is also possible to extract this info from the Nuxt data on the website,
 181         # however that seems unreliable - sometimes it randomly doesn't return the data,
 182         # at least when using a non-residential IP.
 183         resp = self._call_api(vid, data={
 184             'operationName': 'vod_VodReaderService_BatchGetVodDetailInfo',
 185             'variables': {
 186                 'params': {
 187                     'vids': [vid],
 188                 },
 189             },
 190             'extensions': {},
 191         })
 192
 193         vod_detail_info = traverse_obj(resp, ('VodDetailInfos', vid), expected_type=dict)
 194         if not vod_detail_info:
 195             raise ExtractorError('This video not found or not available anymore', expected=True)
 196         vod_info = vod_detail_info.get('vodInfo')
 197         title = vod_info.get('title')
 198
 199         if try_get(vod_info, lambda x: x['playbackRights']['playbackRights'] != 'Normal'):
 200             playback_rights_setting = vod_info['playbackRights']['playbackRightsSetting']
 201             if playback_rights_setting == 'SubscriberOnly':
 202                 raise ExtractorError('This video is only available for subscribers', expected=True)
 203             else:
 204                 raise ExtractorError(f'This video is not available ({playback_rights_setting})', expected=True)
 205
 206         language = vod_info.get('languageName')
 207         formats = []
 208         for play_info in (vod_info.get('playInfos') or []):
 209             play_url = play_info.get('playUrl')
 210             if not play_url:
 211                 continue
 212             format_id = play_info.get('desc')
 213             formats.append({
 214                 'ext': 'mp4',
 215                 'filesize': int_or_none(play_info.get('fileSize')),
 216                 'format_id': format_id,
 217                 'height': int_or_none(format_id[:-1]) if format_id else None,
 218                 'language': language,
 219                 'protocol': 'm3u8_native',
 220                 'tbr': int_or_none(play_info.get('bitrate')),
 221                 'url': play_url,
 222                 'http_headers': self._HEADERS,
 223             })
 224
 225         category = vod_info.get('categoryName')
 226         get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))
 227
 228         info = {
 229             'id': vid,
 230             'title': title,
 231             'formats': formats,
 232             'thumbnail': vod_info.get('coverUrl'),
 233             'timestamp': int_or_none(vod_info.get('publishTs')),
 234             'duration': int_or_none(vod_info.get('duration')),
 235             'view_count': get_count('watch'),
 236             'like_count': get_count('like'),
 237             'comment_count': get_count('comment'),
 238             'categories': [category] if category else None,
 239             '__post_extractor': self.extract_comments(vid),
 240         }
 241         info.update(self._extract_streamer_info(vod_detail_info))
 242         return info
 243
 244     def _get_comments(self, vid):
 245         for page in itertools.count(1):
 246             comments_json = self._call_api(vid, data={
 247                 'operationName': 'public_CommentProxyService_GetCommentList',
 248                 'variables': {
 249                     'params': {
 250                         'appInfo': {
 251                             'postID': vid,
 252                         },
 253                         'preview': {},
 254                         'pageSize': 99,
 255                         'page': page,
 256                     },
 257                 },
 258                 'extensions': {
 259                     'singleReq': 'true',
 260                 },
 261             })
 262             for comment in comments_json['commentList']:
 263                 content = comment.get('content')
 264                 if not content:
 265                     continue
 266                 author = comment.get('author') or {}
 267                 parent = comment.get('parentID')
 268                 yield {
 269                     'author': author.get('nickName'),
 270                     'author_id': str_or_none(author.get('uid')),
 271                     'id': str_or_none(comment.get('commentID')),
 272                     'text': content,
 273                     'timestamp': int_or_none(comment.get('createdAt')),
 274                     'parent': 'root' if parent == 0 else str_or_none(parent),
 275                 }
 276
 277             if comments_json['lastPage']:
 278                 break
 279
 280
 281 class TrovoChannelBaseIE(TrovoBaseIE):
 282     def _entries(self, spacename):
 283         for page in itertools.count(1):
 284             vod_json = self._call_api(spacename, data={
 285                 'operationName': self._OPERATION,
 286                 'variables': {
 287                     'params': {
 288                         'terminalSpaceID': {
 289                             'spaceName': spacename,
 290                         },
 291                         'currPage': page,
 292                         'pageSize': 99,
 293                     },
 294                 },
 295                 'extensions': {
 296                     'singleReq': 'true',
 297                 },
 298             })
 299             vods = vod_json.get('vodInfos', [])
 300             for vod in vods:
 301                 vid = vod.get('vid')
 302                 room = traverse_obj(vod, ('spaceInfo', 'roomID'))
 303                 yield self.url_result(
 304                     f'https://trovo.live/s/{spacename}/{room}?vid={vid}',
 305                     ie=TrovoVodIE.ie_key())
 306             has_more = vod_json.get('hasMore')
 307             if not has_more:
 308                 break
 309
 310     def _real_extract(self, url):
 311         spacename = self._match_id(url)
 312         return self.playlist_result(self._entries(spacename), playlist_id=spacename)
 313
 314
 315 class TrovoChannelVodIE(TrovoChannelBaseIE):
 316     _VALID_URL = r'trovovod:(?P<id>[^\s]+)'
 317     IE_DESC = 'All VODs of a trovo.live channel; "trovovod:" prefix'
 318
 319     _TESTS = [{
 320         'url': 'trovovod:OneTappedYou',
 321         'playlist_mincount': 24,
 322         'info_dict': {
 323             'id': 'OneTappedYou',
 324         },
 325     }]
 326
 327     _OPERATION = 'vod_VodReaderService_GetChannelLtvVideoInfos'
 328
 329
 330 class TrovoChannelClipIE(TrovoChannelBaseIE):
 331     _VALID_URL = r'trovoclip:(?P<id>[^\s]+)'
 332     IE_DESC = 'All Clips of a trovo.live channel; "trovoclip:" prefix'
 333
 334     _TESTS = [{
 335         'url': 'trovoclip:OneTappedYou',
 336         'playlist_mincount': 29,
 337         'info_dict': {
 338             'id': 'OneTappedYou',
 339         },
 340     }]
 341
 342     _OPERATION = 'vod_VodReaderService_GetChannelClipVideoInfos'