yt_dlp/extractor/funimation.py

   1 import random
   2 import re
   3 import string
   4
   5 from .common import InfoExtractor
   6 from ..networking.exceptions import HTTPError
   7 from ..utils import (
   8     ExtractorError,
   9     determine_ext,
  10     int_or_none,
  11     join_nonempty,
  12     js_to_json,
  13     make_archive_id,
  14     orderedSet,
  15     qualities,
  16     str_or_none,
  17     traverse_obj,
  18     try_get,
  19     urlencode_postdata,
  20 )
  21
  22
  23 class FunimationBaseIE(InfoExtractor):
  24     _NETRC_MACHINE = 'funimation'
  25     _REGION = None
  26     _TOKEN = None
  27
  28     def _get_region(self):
  29         region_cookie = self._get_cookies('https://www.funimation.com').get('region')
  30         region = region_cookie.value if region_cookie else self.get_param('geo_bypass_country')
  31         return region or traverse_obj(
  32             self._download_json(
  33                 'https://geo-service.prd.funimationsvc.com/geo/v1/region/check', None, fatal=False,
  34                 note='Checking geo-location', errnote='Unable to fetch geo-location information'),
  35             'region') or 'US'
  36
  37     def _perform_login(self, username, password):
  38         if self._TOKEN:
  39             return
  40         try:
  41             data = self._download_json(
  42                 'https://prod-api-funimationnow.dadcdigital.com/api/auth/login/',
  43                 None, 'Logging in', data=urlencode_postdata({
  44                     'username': username,
  45                     'password': password,
  46                 }))
  47             FunimationBaseIE._TOKEN = data['token']
  48         except ExtractorError as e:
  49             if isinstance(e.cause, HTTPError) and e.cause.status == 401:
  50                 error = self._parse_json(e.cause.response.read().decode(), None)['error']
  51                 raise ExtractorError(error, expected=True)
  52             raise
  53
  54
  55 class FunimationPageIE(FunimationBaseIE):
  56     IE_NAME = 'funimation:page'
  57     _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:(?P<lang>[^/]+)/)?(?:shows|v)/(?P<show>[^/]+)/(?P<episode>[^/?#&]+)'
  58
  59     _TESTS = [{
  60         'url': 'https://www.funimation.com/shows/attack-on-titan-junior-high/broadcast-dub-preview/',
  61         'info_dict': {
  62             'id': '210050',
  63             'ext': 'mp4',
  64             'title': 'Broadcast Dub Preview',
  65             # Other metadata is tested in FunimationIE
  66         },
  67         'params': {
  68             'skip_download': 'm3u8',
  69         },
  70         'add_ie': ['Funimation'],
  71     }, {
  72         # Not available in US
  73         'url': 'https://www.funimation.com/shows/hacksign/role-play/',
  74         'only_matching': True,
  75     }, {
  76         # with lang code
  77         'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
  78         'only_matching': True,
  79     }, {
  80         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
  81         'only_matching': True,
  82     }, {
  83         'url': 'https://www.funimation.com/v/a-certain-scientific-railgun/super-powered-level-5',
  84         'only_matching': True,
  85     }]
  86
  87     def _real_initialize(self):
  88         if not self._REGION:
  89             FunimationBaseIE._REGION = self._get_region()
  90
  91     def _real_extract(self, url):
  92         locale, show, episode = self._match_valid_url(url).group('lang', 'show', 'episode')
  93
  94         video_id = traverse_obj(self._download_json(
  95             f'https://title-api.prd.funimationsvc.com/v1/shows/{show}/episodes/{episode}',
  96             f'{show}_{episode}', query={
  97                 'deviceType': 'web',
  98                 'region': self._REGION,
  99                 'locale': locale or 'en',
 100             }), ('videoList', ..., 'id'), get_all=False)
 101
 102         return self.url_result(f'https://www.funimation.com/player/{video_id}', FunimationIE.ie_key(), video_id)
 103
 104
 105 class FunimationIE(FunimationBaseIE):
 106     _VALID_URL = r'https?://(?:www\.)?funimation\.com/player/(?P<id>\d+)'
 107
 108     _TESTS = [{
 109         'url': 'https://www.funimation.com/player/210051',
 110         'info_dict': {
 111             'id': '210050',
 112             'display_id': 'broadcast-dub-preview',
 113             'ext': 'mp4',
 114             'title': 'Broadcast Dub Preview',
 115             'thumbnail': r're:https?://.*\.(?:jpg|png)',
 116             'episode': 'Broadcast Dub Preview',
 117             'episode_id': '210050',
 118             'season': 'Extras',
 119             'season_id': '166038',
 120             'season_number': 99,
 121             'series': 'Attack on Titan: Junior High',
 122             'description': '',
 123             'duration': 155,
 124         },
 125         'params': {
 126             'skip_download': 'm3u8',
 127         },
 128     }, {
 129         'note': 'player_id should be extracted with the relevent compat-opt',
 130         'url': 'https://www.funimation.com/player/210051',
 131         'info_dict': {
 132             'id': '210051',
 133             'display_id': 'broadcast-dub-preview',
 134             'ext': 'mp4',
 135             'title': 'Broadcast Dub Preview',
 136             'thumbnail': r're:https?://.*\.(?:jpg|png)',
 137             'episode': 'Broadcast Dub Preview',
 138             'episode_id': '210050',
 139             'season': 'Extras',
 140             'season_id': '166038',
 141             'season_number': 99,
 142             'series': 'Attack on Titan: Junior High',
 143             'description': '',
 144             'duration': 155,
 145         },
 146         'params': {
 147             'skip_download': 'm3u8',
 148             'compat_opts': ['seperate-video-versions'],
 149         },
 150     }]
 151
 152     @staticmethod
 153     def _get_experiences(episode):
 154         for lang, lang_data in episode.get('languages', {}).items():
 155             for video_data in lang_data.values():
 156                 for version, f in video_data.items():
 157                     yield lang, version.title(), f
 158
 159     def _get_episode(self, webpage, experience_id=None, episode_id=None, fatal=True):
 160         """ Extract the episode, season and show objects given either episode/experience id """
 161         show = self._parse_json(
 162             self._search_regex(
 163                 r'show\s*=\s*({.+?})\s*;', webpage, 'show data', fatal=fatal),
 164             experience_id, transform_source=js_to_json, fatal=fatal) or []
 165         for season in show.get('seasons', []):
 166             for episode in season.get('episodes', []):
 167                 if episode_id is not None:
 168                     if str(episode.get('episodePk')) == episode_id:
 169                         return episode, season, show
 170                     continue
 171                 for _, _, f in self._get_experiences(episode):
 172                     if f.get('experienceId') == experience_id:
 173                         return episode, season, show
 174         if fatal:
 175             raise ExtractorError('Unable to find episode information')
 176         else:
 177             self.report_warning('Unable to find episode information')
 178         return {}, {}, {}
 179
 180     def _real_extract(self, url):
 181         initial_experience_id = self._match_id(url)
 182         webpage = self._download_webpage(
 183             url, initial_experience_id, note=f'Downloading player webpage for {initial_experience_id}')
 184         episode, season, show = self._get_episode(webpage, experience_id=int(initial_experience_id))
 185         episode_id = str(episode['episodePk'])
 186         display_id = episode.get('slug') or episode_id
 187
 188         formats, subtitles, thumbnails, duration = [], {}, [], 0
 189         requested_languages, requested_versions = self._configuration_arg('language'), self._configuration_arg('version')
 190         language_preference = qualities((requested_languages or [''])[::-1])
 191         source_preference = qualities((requested_versions or ['uncut', 'simulcast'])[::-1])
 192         only_initial_experience = 'seperate-video-versions' in self.get_param('compat_opts', [])
 193
 194         for lang, version, fmt in self._get_experiences(episode):
 195             experience_id = str(fmt['experienceId'])
 196             if ((only_initial_experience and experience_id != initial_experience_id)
 197                     or (requested_languages and lang.lower() not in requested_languages)
 198                     or (requested_versions and version.lower() not in requested_versions)):
 199                 continue
 200             thumbnails.append({'url': fmt.get('poster')})
 201             duration = max(duration, fmt.get('duration', 0))
 202             format_name = f'{version} {lang} ({experience_id})'
 203             self.extract_subtitles(
 204                 subtitles, experience_id, display_id=display_id, format_name=format_name,
 205                 episode=episode if experience_id == initial_experience_id else episode_id)
 206
 207             headers = {}
 208             if self._TOKEN:
 209                 headers['Authorization'] = f'Token {self._TOKEN}'
 210             page = self._download_json(
 211                 f'https://www.funimation.com/api/showexperience/{experience_id}/',
 212                 display_id, headers=headers, expected_status=403, query={
 213                     'pinst_id': ''.join(random.choices(string.digits + string.ascii_letters, k=8)),
 214                 }, note=f'Downloading {format_name} JSON')
 215             sources = page.get('items') or []
 216             if not sources:
 217                 error = try_get(page, lambda x: x['errors'][0], dict)
 218                 if error:
 219                     self.report_warning('{} said: Error {} - {}'.format(
 220                         self.IE_NAME, error.get('code'), error.get('detail') or error.get('title')))
 221                 else:
 222                     self.report_warning('No sources found for format')
 223
 224             current_formats = []
 225             for source in sources:
 226                 source_url = source.get('src')
 227                 source_type = source.get('videoType') or determine_ext(source_url)
 228                 if source_type == 'm3u8':
 229                     current_formats.extend(self._extract_m3u8_formats(
 230                         source_url, display_id, 'mp4', m3u8_id='{}-{}'.format(experience_id, 'hls'), fatal=False,
 231                         note=f'Downloading {format_name} m3u8 information'))
 232                 else:
 233                     current_formats.append({
 234                         'format_id': f'{experience_id}-{source_type}',
 235                         'url': source_url,
 236                     })
 237                 for f in current_formats:
 238                     # TODO: Convert language to code
 239                     f.update({
 240                         'language': lang,
 241                         'format_note': version,
 242                         'source_preference': source_preference(version.lower()),
 243                         'language_preference': language_preference(lang.lower()),
 244                     })
 245                 formats.extend(current_formats)
 246         if not formats and (requested_languages or requested_versions):
 247             self.raise_no_formats(
 248                 'There are no video formats matching the requested languages/versions', expected=True, video_id=display_id)
 249         self._remove_duplicate_formats(formats)
 250
 251         return {
 252             'id': episode_id,
 253             '_old_archive_ids': [make_archive_id(self, initial_experience_id)],
 254             'display_id': display_id,
 255             'duration': duration,
 256             'title': episode['episodeTitle'],
 257             'description': episode.get('episodeSummary'),
 258             'episode': episode.get('episodeTitle'),
 259             'episode_number': int_or_none(episode.get('episodeId')),
 260             'episode_id': episode_id,
 261             'season': season.get('seasonTitle'),
 262             'season_number': int_or_none(season.get('seasonId')),
 263             'season_id': str_or_none(season.get('seasonPk')),
 264             'series': show.get('showTitle'),
 265             'formats': formats,
 266             'thumbnails': thumbnails,
 267             'subtitles': subtitles,
 268             '_format_sort_fields': ('lang', 'source'),
 269         }
 270
 271     def _get_subtitles(self, subtitles, experience_id, episode, display_id, format_name):
 272         if isinstance(episode, str):
 273             webpage = self._download_webpage(
 274                 f'https://www.funimation.com/player/{experience_id}/', display_id,
 275                 fatal=False, note=f'Downloading player webpage for {format_name}')
 276             episode, _, _ = self._get_episode(webpage, episode_id=episode, fatal=False)
 277
 278         for _, version, f in self._get_experiences(episode):
 279             for source in f.get('sources'):
 280                 for text_track in source.get('textTracks'):
 281                     if not text_track.get('src'):
 282                         continue
 283                     sub_type = text_track.get('type').upper()
 284                     sub_type = sub_type if sub_type != 'FULL' else None
 285                     current_sub = {
 286                         'url': text_track['src'],
 287                         'name': join_nonempty(version, text_track.get('label'), sub_type, delim=' '),
 288                     }
 289                     lang = join_nonempty(text_track.get('language', 'und'),
 290                                          version if version != 'Simulcast' else None,
 291                                          sub_type, delim='_')
 292                     if current_sub not in subtitles.get(lang, []):
 293                         subtitles.setdefault(lang, []).append(current_sub)
 294         return subtitles
 295
 296
 297 class FunimationShowIE(FunimationBaseIE):
 298     IE_NAME = 'funimation:show'
 299     _VALID_URL = r'(?P<url>https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?P<locale>[^/]+)?/?shows/(?P<id>[^/?#&]+))/?(?:[?#]|$)'
 300
 301     _TESTS = [{
 302         'url': 'https://www.funimation.com/en/shows/sk8-the-infinity',
 303         'info_dict': {
 304             'id': '1315000',
 305             'title': 'SK8 the Infinity',
 306         },
 307         'playlist_count': 13,
 308         'params': {
 309             'skip_download': True,
 310         },
 311     }, {
 312         # without lang code
 313         'url': 'https://www.funimation.com/shows/ouran-high-school-host-club/',
 314         'info_dict': {
 315             'id': '39643',
 316             'title': 'Ouran High School Host Club',
 317         },
 318         'playlist_count': 26,
 319         'params': {
 320             'skip_download': True,
 321         },
 322     }]
 323
 324     def _real_initialize(self):
 325         if not self._REGION:
 326             FunimationBaseIE._REGION = self._get_region()
 327
 328     def _real_extract(self, url):
 329         base_url, locale, display_id = self._match_valid_url(url).groups()
 330
 331         show_info = self._download_json(
 332             'https://title-api.prd.funimationsvc.com/v2/shows/{}?region={}&deviceType=web&locale={}'.format(
 333                 display_id, self._REGION, locale or 'en'), display_id)
 334         items_info = self._download_json(
 335             'https://prod-api-funimationnow.dadcdigital.com/api/funimation/episodes/?limit=99999&title_id={}'.format(
 336                 show_info.get('id')), display_id)
 337
 338         vod_items = traverse_obj(items_info, ('items', ..., lambda k, _: re.match(r'(?i)mostRecent[AS]vod', k), 'item'))
 339
 340         return {
 341             '_type': 'playlist',
 342             'id': str_or_none(show_info['id']),
 343             'title': show_info['name'],
 344             'entries': orderedSet(
 345                 self.url_result(
 346                     '{}/{}'.format(base_url, vod_item.get('episodeSlug')), FunimationPageIE.ie_key(),
 347                     vod_item.get('episodeId'), vod_item.get('episodeName'))
 348                 for vod_item in sorted(vod_items, key=lambda x: x.get('episodeOrder', -1))),
 349         }