yt_dlp/extractor/wrestleuniverse.py

   1 import base64
   2 import binascii
   3 import json
   4 import time
   5 import uuid
   6
   7 from .common import InfoExtractor
   8 from ..dependencies import Cryptodome
   9 from ..utils import (
  10     ExtractorError,
  11     int_or_none,
  12     jwt_decode_hs256,
  13     traverse_obj,
  14     try_call,
  15     url_basename,
  16     url_or_none,
  17     urlencode_postdata,
  18     variadic,
  19 )
  20
  21
  22 class WrestleUniverseBaseIE(InfoExtractor):
  23     _NETRC_MACHINE = 'wrestleuniverse'
  24     _VALID_URL_TMPL = r'https?://(?:www\.)?wrestle-universe\.com/(?:(?P<lang>\w{2})/)?%s/(?P<id>\w+)'
  25     _API_HOST = 'api.wrestle-universe.com'
  26     _API_PATH = None
  27     _REAL_TOKEN = None
  28     _TOKEN_EXPIRY = None
  29     _REFRESH_TOKEN = None
  30     _DEVICE_ID = None
  31     _LOGIN_QUERY = {'key': 'AIzaSyCaRPBsDQYVDUWWBXjsTrHESi2r_F3RAdA'}
  32     _LOGIN_HEADERS = {
  33         'Accept': '*/*',
  34         'Content-Type': 'application/json',
  35         'X-Client-Version': 'Chrome/JsCore/9.9.4/FirebaseCore-web',
  36         'X-Firebase-gmpid': '1:307308870738:web:820f38fe5150c8976e338b',
  37         'Referer': 'https://www.wrestle-universe.com/',
  38         'Origin': 'https://www.wrestle-universe.com',
  39     }
  40
  41     @property
  42     def _TOKEN(self):
  43         if not self._REAL_TOKEN or not self._TOKEN_EXPIRY:
  44             token = try_call(lambda: self._get_cookies('https://www.wrestle-universe.com/')['token'].value)
  45             if not token and not self._REFRESH_TOKEN:
  46                 self.raise_login_required()
  47             self._TOKEN = token
  48
  49         if not self._REAL_TOKEN or self._TOKEN_EXPIRY <= int(time.time()):
  50             if not self._REFRESH_TOKEN:
  51                 raise ExtractorError(
  52                     'Expired token. Refresh your cookies in browser and try again', expected=True)
  53             self._refresh_token()
  54
  55         return self._REAL_TOKEN
  56
  57     @_TOKEN.setter
  58     def _TOKEN(self, value):
  59         self._REAL_TOKEN = value
  60
  61         expiry = traverse_obj(value, ({jwt_decode_hs256}, 'exp', {int_or_none}))
  62         if not expiry:
  63             raise ExtractorError('There was a problem with the auth token')
  64         self._TOKEN_EXPIRY = expiry
  65
  66     def _perform_login(self, username, password):
  67         login = self._download_json(
  68             'https://identitytoolkit.googleapis.com/v1/accounts:signInWithPassword', None,
  69             'Logging in', query=self._LOGIN_QUERY, headers=self._LOGIN_HEADERS, data=json.dumps({
  70                 'returnSecureToken': True,
  71                 'email': username,
  72                 'password': password,
  73             }, separators=(',', ':')).encode(), expected_status=400)
  74         token = traverse_obj(login, ('idToken', {str}))
  75         if not token:
  76             raise ExtractorError(
  77                 f'Unable to log in: {traverse_obj(login, ("error", "message"))}', expected=True)
  78         self._REFRESH_TOKEN = traverse_obj(login, ('refreshToken', {str}))
  79         if not self._REFRESH_TOKEN:
  80             self.report_warning('No refresh token was granted')
  81         self._TOKEN = token
  82
  83     def _real_initialize(self):
  84         if self._DEVICE_ID:
  85             return
  86
  87         self._DEVICE_ID = self._configuration_arg('device_id', [None], ie_key=self._NETRC_MACHINE)[0]
  88         if not self._DEVICE_ID:
  89             self._DEVICE_ID = self.cache.load(self._NETRC_MACHINE, 'device_id')
  90             if self._DEVICE_ID:
  91                 return
  92             self._DEVICE_ID = str(uuid.uuid4())
  93
  94         self.cache.store(self._NETRC_MACHINE, 'device_id', self._DEVICE_ID)
  95
  96     def _refresh_token(self):
  97         refresh = self._download_json(
  98             'https://securetoken.googleapis.com/v1/token', None, 'Refreshing token',
  99             query=self._LOGIN_QUERY, data=urlencode_postdata({
 100                 'grant_type': 'refresh_token',
 101                 'refresh_token': self._REFRESH_TOKEN,
 102             }), headers={
 103                 **self._LOGIN_HEADERS,
 104                 'Content-Type': 'application/x-www-form-urlencoded',
 105             })
 106         if traverse_obj(refresh, ('refresh_token', {str})):
 107             self._REFRESH_TOKEN = refresh['refresh_token']
 108         token = traverse_obj(refresh, 'access_token', 'id_token', expected_type=str)
 109         if not token:
 110             raise ExtractorError('No auth token returned from refresh request')
 111         self._TOKEN = token
 112
 113     def _call_api(self, video_id, param='', msg='API', auth=True, data=None, query={}, fatal=True):
 114         headers = {'CA-CID': ''}
 115         if data:
 116             headers['Content-Type'] = 'application/json;charset=utf-8'
 117             data = json.dumps(data, separators=(',', ':')).encode()
 118         if auth and self._TOKEN:
 119             headers['Authorization'] = f'Bearer {self._TOKEN}'
 120         return self._download_json(
 121             f'https://{self._API_HOST}/v1/{self._API_PATH}/{video_id}{param}', video_id,
 122             note=f'Downloading {msg} JSON', errnote=f'Failed to download {msg} JSON',
 123             data=data, headers=headers, query=query, fatal=fatal)
 124
 125     def _call_encrypted_api(self, video_id, param='', msg='API', data={}, query={}, fatal=True):
 126         if not Cryptodome.RSA:
 127             raise ExtractorError('pycryptodomex not found. Please install', expected=True)
 128         private_key = Cryptodome.RSA.generate(2048)
 129         cipher = Cryptodome.PKCS1_OAEP.new(private_key, hashAlgo=Cryptodome.SHA1)
 130
 131         def decrypt(data):
 132             if not data:
 133                 return None
 134             try:
 135                 return cipher.decrypt(base64.b64decode(data)).decode()
 136             except (ValueError, binascii.Error) as e:
 137                 raise ExtractorError(f'Could not decrypt data: {e}')
 138
 139         token = base64.b64encode(private_key.public_key().export_key('DER')).decode()
 140         api_json = self._call_api(video_id, param, msg, data={
 141             'deviceId': self._DEVICE_ID,
 142             'token': token,
 143             **data,
 144         }, query=query, fatal=fatal)
 145         return api_json, decrypt
 146
 147     def _download_metadata(self, url, video_id, lang, props_keys):
 148         metadata = self._call_api(video_id, msg='metadata', query={'al': lang or 'ja'}, auth=False, fatal=False)
 149         if not metadata:
 150             webpage = self._download_webpage(url, video_id)
 151             nextjs_data = self._search_nextjs_data(webpage, video_id, fatal=False)
 152             metadata = traverse_obj(nextjs_data, (
 153                 'props', 'pageProps', *variadic(props_keys, (str, bytes, dict, set)), {dict})) or {}
 154         return metadata
 155
 156     def _get_formats(self, data, path, video_id=None):
 157         hls_url = traverse_obj(data, path, get_all=False)
 158         if not hls_url and not data.get('canWatch'):
 159             self.raise_no_formats(
 160                 'This account does not have access to the requested content', expected=True)
 161         elif not hls_url:
 162             self.raise_no_formats('No supported formats found')
 163         return self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls', live=True)
 164
 165
 166 class WrestleUniverseVODIE(WrestleUniverseBaseIE):
 167     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'videos'
 168     _TESTS = [{
 169         'url': 'https://www.wrestle-universe.com/en/videos/dp8mpjmcKfxzUhEHM2uFws',
 170         'info_dict': {
 171             'id': 'dp8mpjmcKfxzUhEHM2uFws',
 172             'ext': 'mp4',
 173             'title': 'The 3rd “Futari wa Princess” Max Heart Tournament',
 174             'description': 'md5:318d5061e944797fbbb81d5c7dd00bf5',
 175             'location': '埼玉・春日部ふれあいキューブ',
 176             'channel': 'tjpw',
 177             'duration': 7119,
 178             'timestamp': 1674979200,
 179             'upload_date': '20230129',
 180             'thumbnail': 'https://image.asset.wrestle-universe.com/8FjD67P8rZc446RBQs5RBN/8FjD67P8rZc446RBQs5RBN',
 181             'chapters': 'count:7',
 182             'cast': 'count:21',
 183         },
 184         'params': {
 185             'skip_download': 'm3u8',
 186         },
 187     }]
 188
 189     _API_PATH = 'videoEpisodes'
 190
 191     def _real_extract(self, url):
 192         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 193         metadata = self._download_metadata(url, video_id, lang, 'videoEpisodeFallbackData')
 194         video_data = self._call_api(video_id, ':watch', 'watch', data={'deviceId': self._DEVICE_ID})
 195
 196         return {
 197             'id': video_id,
 198             'formats': self._get_formats(video_data, ('protocolHls', 'url', {url_or_none}), video_id),
 199             **traverse_obj(metadata, {
 200                 'title': ('displayName', {str}),
 201                 'description': ('description', {str}),
 202                 'channel': ('labels', 'group', {str}),
 203                 'location': ('labels', 'venue', {str}),
 204                 'timestamp': ('watchStartTime', {int_or_none}),
 205                 'thumbnail': ('keyVisualUrl', {url_or_none}),
 206                 'cast': ('casts', ..., 'displayName', {str}),
 207                 'duration': ('duration', {int}),
 208                 'chapters': ('videoChapters', lambda _, v: isinstance(v.get('start'), int), {
 209                     'title': ('displayName', {str}),
 210                     'start_time': ('start', {int}),
 211                     'end_time': ('end', {int}),
 212                 }),
 213             }),
 214         }
 215
 216
 217 class WrestleUniversePPVIE(WrestleUniverseBaseIE):
 218     _VALID_URL = WrestleUniverseBaseIE._VALID_URL_TMPL % 'lives'
 219     _TESTS = [{
 220         'note': 'HLS AES-128 key obtained via API',
 221         'url': 'https://www.wrestle-universe.com/en/lives/buH9ibbfhdJAY4GKZcEuJX',
 222         'info_dict': {
 223             'id': 'buH9ibbfhdJAY4GKZcEuJX',
 224             'ext': 'mp4',
 225             'title': '【PPV】Beyond the origins, into the future',
 226             'description': 'md5:9a872db68cd09be4a1e35a3ee8b0bdfc',
 227             'channel': 'tjpw',
 228             'location': '東京・Twin Box AKIHABARA',
 229             'duration': 10098,
 230             'timestamp': 1675076400,
 231             'upload_date': '20230130',
 232             'thumbnail': 'https://image.asset.wrestle-universe.com/rJs2m7cBaLXrwCcxMdQGRM/rJs2m7cBaLXrwCcxMdQGRM',
 233             'thumbnails': 'count:3',
 234             'hls_aes': {
 235                 'key': '5633184acd6e43f1f1ac71c6447a4186',
 236                 'iv': '5bac71beb33197d5600337ce86de7862',
 237             },
 238         },
 239         'params': {
 240             'skip_download': 'm3u8',
 241         },
 242         'skip': 'No longer available',
 243     }, {
 244         'note': 'unencrypted HLS',
 245         'url': 'https://www.wrestle-universe.com/en/lives/wUG8hP5iApC63jbtQzhVVx',
 246         'info_dict': {
 247             'id': 'wUG8hP5iApC63jbtQzhVVx',
 248             'ext': 'mp4',
 249             'title': 'GRAND PRINCESS \'22',
 250             'description': 'md5:e4f43d0d4262de3952ff34831bc99858',
 251             'channel': 'tjpw',
 252             'location': '東京・両国国技館',
 253             'duration': 18044,
 254             'timestamp': 1647665400,
 255             'upload_date': '20220319',
 256             'thumbnail': 'https://image.asset.wrestle-universe.com/i8jxSTCHPfdAKD4zN41Psx/i8jxSTCHPfdAKD4zN41Psx',
 257             'thumbnails': 'count:3',
 258         },
 259         'params': {
 260             'skip_download': 'm3u8',
 261         },
 262     }, {
 263         'note': 'manifest provides live-a (partial) and live-b (full) streams',
 264         'url': 'https://www.wrestle-universe.com/en/lives/umc99R9XsexXrxr9VjTo9g',
 265         'only_matching': True,
 266     }]
 267
 268     _API_PATH = 'events'
 269
 270     def _real_extract(self, url):
 271         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 272         metadata = self._download_metadata(url, video_id, lang, 'eventFallbackData')
 273
 274         info = {
 275             'id': video_id,
 276             **traverse_obj(metadata, {
 277                 'title': ('displayName', {str}),
 278                 'description': ('description', {str}),
 279                 'channel': ('labels', 'group', {str}),
 280                 'location': ('labels', 'venue', {str}),
 281                 'timestamp': ('startTime', {int_or_none}),
 282                 'thumbnails': (('keyVisualUrl', 'alterKeyVisualUrl', 'heroKeyVisualUrl'), {'url': {url_or_none}}),
 283             }),
 284         }
 285
 286         ended_time = traverse_obj(metadata, ('endedTime', {int_or_none}))
 287         if info.get('timestamp') and ended_time:
 288             info['duration'] = ended_time - info['timestamp']
 289
 290         video_data, decrypt = self._call_encrypted_api(
 291             video_id, ':watchArchive', 'watch archive', data={'method': 1})
 292         # 'chromecastUrls' can be only partial videos, avoid
 293         info['formats'] = self._get_formats(video_data, ('hls', (('urls', ...), 'url'), {url_or_none}), video_id)
 294         for f in info['formats']:
 295             # bitrates are exaggerated in PPV playlists, so avoid wrong/huge filesize_approx values
 296             if f.get('tbr'):
 297                 f['tbr'] = int(f['tbr'] / 2.5)
 298             # prefer variants with the same basename as the master playlist to avoid partial streams
 299             f['format_id'] = url_basename(f['url']).partition('.')[0]
 300             if not f['format_id'].startswith(url_basename(f['manifest_url']).partition('.')[0]):
 301                 f['preference'] = -10
 302
 303         hls_aes_key = traverse_obj(video_data, ('hls', 'key', {decrypt}))
 304         if hls_aes_key:
 305             info['hls_aes'] = {
 306                 'key': hls_aes_key,
 307                 'iv': traverse_obj(video_data, ('hls', 'iv', {decrypt})),
 308             }
 309         elif traverse_obj(video_data, ('hls', 'encryptType', {int})):
 310             self.report_warning('HLS AES-128 key was not found in API response')
 311
 312         return info