yt_dlp/extractor/adn.py

   1 import base64
   2 import binascii
   3 import json
   4 import os
   5 import random
   6 import time
   7
   8 from .common import InfoExtractor
   9 from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
  10 from ..networking.exceptions import HTTPError
  11 from ..utils import (
  12     ExtractorError,
  13     ass_subtitles_timecode,
  14     bytes_to_intlist,
  15     bytes_to_long,
  16     float_or_none,
  17     int_or_none,
  18     intlist_to_bytes,
  19     join_nonempty,
  20     long_to_bytes,
  21     parse_iso8601,
  22     pkcs1pad,
  23     str_or_none,
  24     strip_or_none,
  25     try_get,
  26     unified_strdate,
  27     urlencode_postdata,
  28 )
  29 from ..utils.traversal import traverse_obj
  30
  31
  32 class ADNBaseIE(InfoExtractor):
  33     IE_DESC = 'Animation Digital Network'
  34     _NETRC_MACHINE = 'animationdigitalnetwork'
  35     _BASE = 'animationdigitalnetwork.fr'
  36     _API_BASE_URL = f'https://gw.api.{_BASE}/'
  37     _PLAYER_BASE_URL = f'{_API_BASE_URL}player/'
  38     _HEADERS = {}
  39     _LOGIN_ERR_MESSAGE = 'Unable to log in'
  40     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
  41     _POS_ALIGN_MAP = {
  42         'start': 1,
  43         'end': 3,
  44     }
  45     _LINE_ALIGN_MAP = {
  46         'middle': 8,
  47         'end': 4,
  48     }
  49
  50
  51 class ADNIE(ADNBaseIE):
  52     _VALID_URL = r'https?://(?:www\.)?animationdigitalnetwork\.com/(?:(?P<lang>de)/)?video/[^/?#]+/(?P<id>\d+)'
  53     _TESTS = [{
  54         'url': 'https://animationdigitalnetwork.com/video/558-fruits-basket/9841-episode-1-a-ce-soir',
  55         'md5': '1c9ef066ceb302c86f80c2b371615261',
  56         'info_dict': {
  57             'id': '9841',
  58             'ext': 'mp4',
  59             'title': 'Fruits Basket - Episode 1',
  60             'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
  61             'series': 'Fruits Basket',
  62             'duration': 1437,
  63             'release_date': '20190405',
  64             'comment_count': int,
  65             'average_rating': float,
  66             'season_number': 1,
  67             'episode': 'À ce soir !',
  68             'episode_number': 1,
  69             'thumbnail': str,
  70             'season': 'Season 1',
  71         },
  72         'skip': 'Only available in French and German speaking Europe',
  73     }, {
  74         'url': 'https://animationdigitalnetwork.com/de/video/973-the-eminence-in-shadow/23550-folge-1',
  75         'md5': '5c5651bf5791fa6fcd7906012b9d94e8',
  76         'info_dict': {
  77             'id': '23550',
  78             'ext': 'mp4',
  79             'episode_number': 1,
  80             'duration': 1417,
  81             'release_date': '20231004',
  82             'series': 'The Eminence in Shadow',
  83             'season_number': 2,
  84             'episode': str,
  85             'title': str,
  86             'thumbnail': str,
  87             'season': 'Season 2',
  88             'comment_count': int,
  89             'average_rating': float,
  90             'description': str,
  91         },
  92         # 'skip': 'Only available in French and German speaking Europe',
  93     }]
  94
  95     def _get_subtitles(self, sub_url, video_id):
  96         if not sub_url:
  97             return None
  98
  99         enc_subtitles = self._download_webpage(
 100             sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
 101         subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
 102         if subtitle_location:
 103             enc_subtitles = self._download_webpage(
 104                 subtitle_location, video_id, 'Downloading subtitles data',
 105                 fatal=False, headers={'Origin': 'https://' + self._BASE})
 106         if not enc_subtitles:
 107             return None
 108
 109         # http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
 110         dec_subtitles = unpad_pkcs7(aes_cbc_decrypt_bytes(
 111             base64.b64decode(enc_subtitles[24:]),
 112             binascii.unhexlify(self._K + '7fac1178830cfe0c'),
 113             base64.b64decode(enc_subtitles[:24])))
 114         subtitles_json = self._parse_json(dec_subtitles.decode(), None, fatal=False)
 115         if not subtitles_json:
 116             return None
 117
 118         subtitles = {}
 119         for sub_lang, sub in subtitles_json.items():
 120             ssa = '''[Script Info]
 121 ScriptType:V4.00
 122 [V4 Styles]
 123 Format: Name,Fontname,Fontsize,PrimaryColour,SecondaryColour,TertiaryColour,BackColour,Bold,Italic,BorderStyle,Outline,Shadow,Alignment,MarginL,MarginR,MarginV,AlphaLevel,Encoding
 124 Style: Default,Arial,18,16777215,16777215,16777215,0,-1,0,1,1,0,2,20,20,20,0,0
 125 [Events]
 126 Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 127             for current in sub:
 128                 start, end, text, line_align, position_align = (
 129                     float_or_none(current.get('startTime')),
 130                     float_or_none(current.get('endTime')),
 131                     current.get('text'), current.get('lineAlign'),
 132                     current.get('positionAlign'))
 133                 if start is None or end is None or text is None:
 134                     continue
 135                 alignment = self._POS_ALIGN_MAP.get(position_align, 2) + self._LINE_ALIGN_MAP.get(line_align, 0)
 136                 ssa += os.linesep + 'Dialogue: Marked=0,{},{},Default,,0,0,0,,{}{}'.format(
 137                     ass_subtitles_timecode(start),
 138                     ass_subtitles_timecode(end),
 139                     '{\\a%d}' % alignment if alignment != 2 else '',
 140                     text.replace('\n', '\\N').replace('<i>', '{\\i1}').replace('</i>', '{\\i0}'))
 141
 142             if sub_lang == 'vostf':
 143                 sub_lang = 'fr'
 144             elif sub_lang == 'vostde':
 145                 sub_lang = 'de'
 146             subtitles.setdefault(sub_lang, []).extend([{
 147                 'ext': 'json',
 148                 'data': json.dumps(sub),
 149             }, {
 150                 'ext': 'ssa',
 151                 'data': ssa,
 152             }])
 153         return subtitles
 154
 155     def _perform_login(self, username, password):
 156         try:
 157             access_token = (self._download_json(
 158                 self._API_BASE_URL + 'authentication/login', None,
 159                 'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
 160                 data=urlencode_postdata({
 161                     'password': password,
 162                     'rememberMe': False,
 163                     'source': 'Web',
 164                     'username': username,
 165                 })) or {}).get('accessToken')
 166             if access_token:
 167                 self._HEADERS['Authorization'] = f'Bearer {access_token}'
 168         except ExtractorError as e:
 169             message = None
 170             if isinstance(e.cause, HTTPError) and e.cause.status == 401:
 171                 resp = self._parse_json(
 172                     e.cause.response.read().decode(), None, fatal=False) or {}
 173                 message = resp.get('message') or resp.get('code')
 174             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 175
 176     def _real_extract(self, url):
 177         lang, video_id = self._match_valid_url(url).group('lang', 'id')
 178         self._HEADERS['X-Target-Distribution'] = lang or 'fr'
 179         video_base_url = self._PLAYER_BASE_URL + f'video/{video_id}/'
 180         player = self._download_json(
 181             video_base_url + 'configuration', video_id,
 182             'Downloading player config JSON metadata',
 183             headers=self._HEADERS)['player']
 184         options = player['options']
 185
 186         user = options['user']
 187         if not user.get('hasAccess'):
 188             start_date = traverse_obj(options, ('video', 'startDate', {str}))
 189             if (parse_iso8601(start_date) or 0) > time.time():
 190                 raise ExtractorError(f'This video is not available yet. Release date: {start_date}', expected=True)
 191             self.raise_login_required('This video requires a subscription', method='password')
 192
 193         token = self._download_json(
 194             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
 195             video_id, 'Downloading access token', headers={
 196                 'X-Player-Refresh-Token': user['refreshToken'],
 197             }, data=b'')['token']
 198
 199         links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
 200         self._K = ''.join(random.choices('0123456789abcdef', k=16))
 201         message = bytes_to_intlist(json.dumps({
 202             'k': self._K,
 203             't': token,
 204         }))
 205
 206         # Sometimes authentication fails for no good reason, retry with
 207         # a different random padding
 208         links_data = None
 209         for _ in range(3):
 210             padded_message = intlist_to_bytes(pkcs1pad(message, 128))
 211             n, e = self._RSA_KEY
 212             encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
 213             authorization = base64.b64encode(encrypted_message).decode()
 214
 215             try:
 216                 links_data = self._download_json(
 217                     links_url, video_id, 'Downloading links JSON metadata', headers={
 218                         'X-Player-Token': authorization,
 219                         **self._HEADERS,
 220                     }, query={
 221                         'freeWithAds': 'true',
 222                         'adaptive': 'false',
 223                         'withMetadata': 'true',
 224                         'source': 'Web',
 225                     })
 226                 break
 227             except ExtractorError as e:
 228                 if not isinstance(e.cause, HTTPError):
 229                     raise e
 230
 231                 if e.cause.status == 401:
 232                     # This usually goes away with a different random pkcs1pad, so retry
 233                     continue
 234
 235                 error = self._parse_json(e.cause.response.read(), video_id)
 236                 message = error.get('message')
 237                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
 238                     self.raise_geo_restricted(msg=message)
 239                 raise ExtractorError(message)
 240         else:
 241             raise ExtractorError('Giving up retrying')
 242
 243         links = links_data.get('links') or {}
 244         metas = links_data.get('metadata') or {}
 245         sub_url = (links.get('subtitles') or {}).get('all')
 246         video_info = links_data.get('video') or {}
 247         title = metas['title']
 248
 249         formats = []
 250         for format_id, qualities in (links.get('streaming') or {}).items():
 251             if not isinstance(qualities, dict):
 252                 continue
 253             for quality, load_balancer_url in qualities.items():
 254                 load_balancer_data = self._download_json(
 255                     load_balancer_url, video_id,
 256                     f'Downloading {format_id} {quality} JSON metadata',
 257                     headers=self._HEADERS,
 258                     fatal=False) or {}
 259                 m3u8_url = load_balancer_data.get('location')
 260                 if not m3u8_url:
 261                     continue
 262                 m3u8_formats = self._extract_m3u8_formats(
 263                     m3u8_url, video_id, 'mp4', 'm3u8_native',
 264                     m3u8_id=format_id, fatal=False)
 265                 if format_id == 'vf':
 266                     for f in m3u8_formats:
 267                         f['language'] = 'fr'
 268                 elif format_id == 'vde':
 269                     for f in m3u8_formats:
 270                         f['language'] = 'de'
 271                 formats.extend(m3u8_formats)
 272
 273         if not formats:
 274             self.raise_login_required('This video requires a subscription', method='password')
 275
 276         video = (self._download_json(
 277             self._API_BASE_URL + f'video/{video_id}', video_id,
 278             'Downloading additional video metadata', fatal=False, headers=self._HEADERS) or {}).get('video') or {}
 279         show = video.get('show') or {}
 280
 281         return {
 282             'id': video_id,
 283             'title': title,
 284             'description': strip_or_none(metas.get('summary') or video.get('summary')),
 285             'thumbnail': video_info.get('image') or player.get('image'),
 286             'formats': formats,
 287             'subtitles': self.extract_subtitles(sub_url, video_id),
 288             'episode': metas.get('subtitle') or video.get('name'),
 289             'episode_number': int_or_none(video.get('shortNumber')),
 290             'series': show.get('title'),
 291             'season_number': int_or_none(video.get('season')),
 292             'duration': int_or_none(video_info.get('duration') or video.get('duration')),
 293             'release_date': unified_strdate(video.get('releaseDate')),
 294             'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
 295             'comment_count': int_or_none(video.get('commentsCount')),
 296         }
 297
 298
 299 class ADNSeasonIE(ADNBaseIE):
 300     _VALID_URL = r'https?://(?:www\.)?animationdigitalnetwork\.com/(?:(?P<lang>de)/)?video/(?P<id>\d+)[^/?#]*/?(?:$|[#?])'
 301     _TESTS = [{
 302         'url': 'https://animationdigitalnetwork.com/video/911-tokyo-mew-mew-new',
 303         'playlist_count': 12,
 304         'info_dict': {
 305             'id': '911',
 306             'title': 'Tokyo Mew Mew New',
 307         },
 308         # 'skip': 'Only available in French end German speaking Europe',
 309     }]
 310
 311     def _real_extract(self, url):
 312         lang, video_show_slug = self._match_valid_url(url).group('lang', 'id')
 313         self._HEADERS['X-Target-Distribution'] = lang or 'fr'
 314         show = self._download_json(
 315             f'{self._API_BASE_URL}show/{video_show_slug}/', video_show_slug,
 316             'Downloading show JSON metadata', headers=self._HEADERS)['show']
 317         show_id = str(show['id'])
 318         episodes = self._download_json(
 319             f'{self._API_BASE_URL}video/show/{show_id}', video_show_slug,
 320             'Downloading episode list', headers=self._HEADERS, query={
 321                 'order': 'asc',
 322                 'limit': '-1',
 323             })
 324
 325         def entries():
 326             for episode_id in traverse_obj(episodes, ('videos', ..., 'id', {str_or_none})):
 327                 yield self.url_result(join_nonempty(
 328                     'https://animationdigitalnetwork.com', lang, 'video',
 329                     video_show_slug, episode_id, delim='/'), ADNIE, episode_id)
 330
 331         return self.playlist_result(entries(), show_id, show.get('title'))