yt_dlp/extractor/allstar.py

   1 import functools
   2 import json
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     ExtractorError,
   7     OnDemandPagedList,
   8     int_or_none,
   9     join_nonempty,
  10     parse_qs,
  11     urljoin,
  12 )
  13 from ..utils.traversal import traverse_obj
  14
  15 _FIELDS = '''
  16     _id
  17     clipImageSource
  18     clipImageThumb
  19     clipLink
  20     clipTitle
  21     createdDate
  22     shareId
  23     user { _id }
  24     username
  25     views'''
  26
  27 _EXTRA_FIELDS = '''
  28     clipLength
  29     clipSizeBytes'''
  30
  31 _QUERIES = {
  32     'clip': '''query ($id: String!) {
  33         video: getClip(clipIdentifier: $id) {
  34             %s %s
  35         }
  36     }''' % (_FIELDS, _EXTRA_FIELDS),  # noqa: UP031
  37     'montage': '''query ($id: String!) {
  38         video: getMontage(clipIdentifier: $id) {
  39             %s
  40         }
  41     }''' % _FIELDS,  # noqa: UP031
  42     'Clips': '''query ($page: Int!, $user: String!, $game: Int) {
  43         videos: clips(search: createdDate, page: $page, user: $user, mobile: false, game: $game) {
  44             data { %s %s }
  45         }
  46     }''' % (_FIELDS, _EXTRA_FIELDS),  # noqa: UP031
  47     'Montages': '''query ($page: Int!, $user: String!) {
  48         videos: montages(search: createdDate, page: $page, user: $user) {
  49             data { %s }
  50         }
  51     }''' % _FIELDS,  # noqa: UP031
  52     'Mobile Clips': '''query ($page: Int!, $user: String!) {
  53         videos: clips(search: createdDate, page: $page, user: $user, mobile: true) {
  54             data { %s %s }
  55         }
  56     }''' % (_FIELDS, _EXTRA_FIELDS),  # noqa: UP031
  57 }
  58
  59
  60 class AllstarBaseIE(InfoExtractor):
  61     @staticmethod
  62     def _parse_video_data(video_data):
  63         def media_url_or_none(path):
  64             return urljoin('https://media.allstar.gg/', path)
  65
  66         info = traverse_obj(video_data, {
  67             'id': ('_id', {str}),
  68             'display_id': ('shareId', {str}),
  69             'title': ('clipTitle', {str}),
  70             'url': ('clipLink', {media_url_or_none}),
  71             'thumbnails': (('clipImageThumb', 'clipImageSource'), {'url': {media_url_or_none}}),
  72             'duration': ('clipLength', {int_or_none}),
  73             'filesize': ('clipSizeBytes', {int_or_none}),
  74             'timestamp': ('createdDate', {int_or_none(scale=1000)}),
  75             'uploader': ('username', {str}),
  76             'uploader_id': ('user', '_id', {str}),
  77             'view_count': ('views', {int_or_none}),
  78         })
  79
  80         if info.get('id') and info.get('url'):
  81             basename = 'clip' if '/clips/' in info['url'] else 'montage'
  82             info['webpage_url'] = f'https://allstar.gg/{basename}?{basename}={info["id"]}'
  83
  84         info.update({
  85             'extractor_key': AllstarIE.ie_key(),
  86             'extractor': AllstarIE.IE_NAME,
  87             'uploader_url': urljoin('https://allstar.gg/u/', info.get('uploader_id')),
  88         })
  89
  90         return info
  91
  92     def _call_api(self, query, variables, path, video_id=None, note=None):
  93         response = self._download_json(
  94             'https://a1.allstar.gg/graphql', video_id, note=note,
  95             headers={'content-type': 'application/json'},
  96             data=json.dumps({'variables': variables, 'query': query}).encode())
  97
  98         errors = traverse_obj(response, ('errors', ..., 'message', {str}))
  99         if errors:
 100             raise ExtractorError('; '.join(errors))
 101
 102         return traverse_obj(response, path)
 103
 104
 105 class AllstarIE(AllstarBaseIE):
 106     _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?P<type>(?:clip|montage))\?(?P=type)=(?P<id>[^/?#&]+)'
 107
 108     _TESTS = [{
 109         'url': 'https://allstar.gg/clip?clip=64482c2da9eec30008a67d1b',
 110         'info_dict': {
 111             'id': '64482c2da9eec30008a67d1b',
 112             'title': '4K on Inferno',
 113             'url': 'md5:66befb5381eef0c9456026386c25fa55',
 114             'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
 115             'uploader': 'chrk.',
 116             'ext': 'mp4',
 117             'duration': 20,
 118             'filesize': 21199257,
 119             'timestamp': 1682451501,
 120             'uploader_id': '62b8bdfc9021052f7905882d',
 121             'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
 122             'upload_date': '20230425',
 123             'view_count': int,
 124         },
 125     }, {
 126         'url': 'https://allstar.gg/clip?clip=8LJLY4JKB',
 127         'info_dict': {
 128             'id': '64a1ec6b887f4c0008dc50b8',
 129             'display_id': '8LJLY4JKB',
 130             'title': 'AK-47 3K on Mirage',
 131             'url': 'md5:dde224fd12f035c0e2529a4ae34c4283',
 132             'ext': 'mp4',
 133             'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
 134             'duration': 16,
 135             'filesize': 30175859,
 136             'timestamp': 1688333419,
 137             'uploader': 'cherokee',
 138             'uploader_id': '62b8bdfc9021052f7905882d',
 139             'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
 140             'upload_date': '20230702',
 141             'view_count': int,
 142         },
 143     }, {
 144         'url': 'https://allstar.gg/montage?montage=643e64089da7e9363e1fa66c',
 145         'info_dict': {
 146             'id': '643e64089da7e9363e1fa66c',
 147             'display_id': 'APQLGM2IMXW',
 148             'title': 'cherokee Rapid Fire Snipers Montage',
 149             'url': 'md5:a3ee356022115db2b27c81321d195945',
 150             'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
 151             'ext': 'mp4',
 152             'timestamp': 1681810448,
 153             'uploader': 'cherokee',
 154             'uploader_id': '62b8bdfc9021052f7905882d',
 155             'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
 156             'upload_date': '20230418',
 157             'view_count': int,
 158         },
 159     }, {
 160         'url': 'https://allstar.gg/montage?montage=RILJMH6QOS',
 161         'info_dict': {
 162             'id': '64a2697372ce3703de29e868',
 163             'display_id': 'RILJMH6QOS',
 164             'title': 'cherokee Rapid Fire Snipers Montage',
 165             'url': 'md5:d5672e6f88579730c2310a80fdbc4030',
 166             'thumbnail': r're:https://media\.allstar\.gg/.+\.(?:png|jpg)$',
 167             'ext': 'mp4',
 168             'timestamp': 1688365434,
 169             'uploader': 'cherokee',
 170             'uploader_id': '62b8bdfc9021052f7905882d',
 171             'uploader_url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d',
 172             'upload_date': '20230703',
 173             'view_count': int,
 174         },
 175     }]
 176
 177     def _real_extract(self, url):
 178         query_id, video_id = self._match_valid_url(url).group('type', 'id')
 179
 180         return self._parse_video_data(
 181             self._call_api(
 182                 _QUERIES.get(query_id), {'id': video_id}, ('data', 'video'), video_id))
 183
 184
 185 class AllstarProfileIE(AllstarBaseIE):
 186     _VALID_URL = r'https?://(?:www\.)?allstar\.gg/(?:profile\?user=|u/)(?P<id>[^/?#&]+)'
 187
 188     _TESTS = [{
 189         'url': 'https://allstar.gg/profile?user=62b8bdfc9021052f7905882d',
 190         'info_dict': {
 191             'id': '62b8bdfc9021052f7905882d-clips',
 192             'title': 'cherokee - Clips',
 193         },
 194         'playlist_mincount': 15,
 195     }, {
 196         'url': 'https://allstar.gg/u/cherokee?game=730&view=Clips',
 197         'info_dict': {
 198             'id': '62b8bdfc9021052f7905882d-clips-730',
 199             'title': 'cherokee - Clips - 730',
 200         },
 201         'playlist_mincount': 15,
 202     }, {
 203         'url': 'https://allstar.gg/u/62b8bdfc9021052f7905882d?view=Montages',
 204         'info_dict': {
 205             'id': '62b8bdfc9021052f7905882d-montages',
 206             'title': 'cherokee - Montages',
 207         },
 208         'playlist_mincount': 4,
 209     }, {
 210         'url': 'https://allstar.gg/profile?user=cherokee&view=Mobile Clips',
 211         'info_dict': {
 212             'id': '62b8bdfc9021052f7905882d-mobile',
 213             'title': 'cherokee - Mobile Clips',
 214         },
 215         'playlist_mincount': 1,
 216     }]
 217
 218     _PAGE_SIZE = 10
 219
 220     def _get_page(self, user_id, display_id, game, query, page_num):
 221         page_num += 1
 222
 223         for video_data in self._call_api(
 224                 query, {
 225                     'user': user_id,
 226                     'page': page_num,
 227                     'game': game,
 228                 }, ('data', 'videos', 'data'), display_id, f'Downloading page {page_num}'):
 229             yield self._parse_video_data(video_data)
 230
 231     def _real_extract(self, url):
 232         display_id = self._match_id(url)
 233         profile_data = self._download_json(
 234             urljoin('https://api.allstar.gg/v1/users/profile/', display_id), display_id)
 235         user_id = traverse_obj(profile_data, ('data', ('_id'), {str}))
 236         if not user_id:
 237             raise ExtractorError('Unable to extract the user id')
 238
 239         username = traverse_obj(profile_data, ('data', 'profile', ('username'), {str}))
 240         url_query = parse_qs(url)
 241         game = traverse_obj(url_query, ('game', 0, {int_or_none}))
 242         query_id = traverse_obj(url_query, ('view', 0), default='Clips')
 243
 244         if query_id not in ('Clips', 'Montages', 'Mobile Clips'):
 245             raise ExtractorError(f'Unsupported playlist URL type {query_id!r}')
 246
 247         return self.playlist_result(
 248             OnDemandPagedList(
 249                 functools.partial(
 250                     self._get_page, user_id, display_id, game, _QUERIES.get(query_id)), self._PAGE_SIZE),
 251             playlist_id=join_nonempty(user_id, query_id.lower().split()[0], game),
 252             playlist_title=join_nonempty((username or display_id), query_id, game, delim=' - '))