yt_dlp/extractor/ivi.py

   1 import json
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..dependencies import Cryptodome
   6 from ..utils import ExtractorError, int_or_none, qualities
   7
   8
   9 class IviIE(InfoExtractor):
  10     IE_DESC = 'ivi.ru'
  11     IE_NAME = 'ivi'
  12     _VALID_URL = r'https?://(?:www\.)?ivi\.(?:ru|tv)/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
  13     _EMBED_REGEX = [r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1']
  14     _GEO_BYPASS = False
  15     _GEO_COUNTRIES = ['RU']
  16     _LIGHT_KEY = b'\xf1\x02\x32\xb7\xbc\x5c\x7a\xe8\xf7\x96\xc1\x33\x2b\x27\xa1\x8c'
  17     _LIGHT_URL = 'https://api.ivi.ru/light/'
  18
  19     _TESTS = [
  20         # Single movie
  21         {
  22             'url': 'http://www.ivi.ru/watch/53141',
  23             'md5': '6ff5be2254e796ed346251d117196cf4',
  24             'info_dict': {
  25                 'id': '53141',
  26                 'ext': 'mp4',
  27                 'title': 'Иван Васильевич меняет профессию',
  28                 'description': 'md5:b924063ea1677c8fe343d8a72ac2195f',
  29                 'duration': 5498,
  30                 'thumbnail': r're:^https?://.*\.jpg$',
  31             },
  32             'skip': 'Only works from Russia',
  33         },
  34         # Serial's series
  35         {
  36             'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/9549',
  37             'md5': '221f56b35e3ed815fde2df71032f4b3e',
  38             'info_dict': {
  39                 'id': '9549',
  40                 'ext': 'mp4',
  41                 'title': 'Двое из ларца - Дело Гольдберга (1 часть)',
  42                 'series': 'Двое из ларца',
  43                 'season': 'Сезон 1',
  44                 'season_number': 1,
  45                 'episode': 'Дело Гольдберга (1 часть)',
  46                 'episode_number': 1,
  47                 'duration': 2655,
  48                 'thumbnail': r're:^https?://.*\.jpg$',
  49             },
  50             'skip': 'Only works from Russia',
  51         },
  52         {
  53             # with MP4-HD720 format
  54             'url': 'http://www.ivi.ru/watch/146500',
  55             'md5': 'd63d35cdbfa1ea61a5eafec7cc523e1e',
  56             'info_dict': {
  57                 'id': '146500',
  58                 'ext': 'mp4',
  59                 'title': 'Кукла',
  60                 'description': 'md5:ffca9372399976a2d260a407cc74cce6',
  61                 'duration': 5599,
  62                 'thumbnail': r're:^https?://.*\.jpg$',
  63             },
  64             'skip': 'Only works from Russia',
  65         },
  66         {
  67             'url': 'https://www.ivi.tv/watch/33560/',
  68             'only_matching': True,
  69         },
  70     ]
  71
  72     # Sorted by quality
  73     _KNOWN_FORMATS = (
  74         'MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi',
  75         'MP4-SHQ', 'MP4-HD720', 'MP4-HD1080')
  76
  77     def _real_extract(self, url):
  78         video_id = self._match_id(url)
  79
  80         data = json.dumps({
  81             'method': 'da.content.get',
  82             'params': [
  83                 video_id, {
  84                     'site': 's%d',
  85                     'referrer': f'http://www.ivi.ru/watch/{video_id}',
  86                     'contentid': video_id,
  87                 },
  88             ],
  89         })
  90
  91         for site in (353, 183):
  92             content_data = (data % site).encode()
  93             if site == 353:
  94                 if not Cryptodome.CMAC:
  95                     continue
  96
  97                 timestamp = (self._download_json(
  98                     self._LIGHT_URL, video_id,
  99                     'Downloading timestamp JSON', data=json.dumps({
 100                         'method': 'da.timestamp.get',
 101                         'params': [],
 102                     }).encode(), fatal=False) or {}).get('result')
 103                 if not timestamp:
 104                     continue
 105
 106                 query = {
 107                     'ts': timestamp,
 108                     'sign': Cryptodome.CMAC.new(self._LIGHT_KEY, timestamp.encode() + content_data,
 109                                                 Cryptodome.Blowfish).hexdigest(),
 110                 }
 111             else:
 112                 query = {}
 113
 114             video_json = self._download_json(
 115                 self._LIGHT_URL, video_id,
 116                 'Downloading video JSON', data=content_data, query=query)
 117
 118             error = video_json.get('error')
 119             if error:
 120                 origin = error.get('origin')
 121                 message = error.get('message') or error.get('user_message')
 122                 extractor_msg = 'Unable to download video %s'
 123                 if origin == 'NotAllowedForLocation':
 124                     self.raise_geo_restricted(message, self._GEO_COUNTRIES)
 125                 elif origin == 'NoRedisValidData':
 126                     extractor_msg = 'Video %s does not exist'
 127                 elif site == 353:
 128                     continue
 129                 elif not Cryptodome.CMAC:
 130                     raise ExtractorError('pycryptodomex not found. Please install', expected=True)
 131                 elif message:
 132                     extractor_msg += ': ' + message
 133                 raise ExtractorError(extractor_msg % video_id, expected=True)
 134             else:
 135                 break
 136
 137         result = video_json['result']
 138         title = result['title']
 139
 140         quality = qualities(self._KNOWN_FORMATS)
 141
 142         formats = []
 143         for f in result.get('files', []):
 144             f_url = f.get('url')
 145             content_format = f.get('content_format')
 146             if not f_url:
 147                 continue
 148             if (not self.get_param('allow_unplayable_formats')
 149                     and ('-MDRM-' in content_format or '-FPS-' in content_format)):
 150                 continue
 151             formats.append({
 152                 'url': f_url,
 153                 'format_id': content_format,
 154                 'quality': quality(content_format),
 155                 'filesize': int_or_none(f.get('size_in_bytes')),
 156             })
 157
 158         compilation = result.get('compilation')
 159         episode = title if compilation else None
 160
 161         title = f'{compilation} - {title}' if compilation is not None else title
 162
 163         thumbnails = [{
 164             'url': preview['url'],
 165             'id': preview.get('content_format'),
 166         } for preview in result.get('preview', []) if preview.get('url')]
 167
 168         webpage = self._download_webpage(url, video_id)
 169
 170         season = self._search_regex(
 171             r'<li[^>]+class="season active"[^>]*><a[^>]+>([^<]+)',
 172             webpage, 'season', default=None)
 173         season_number = int_or_none(self._search_regex(
 174             r'<li[^>]+class="season active"[^>]*><a[^>]+data-season(?:-index)?="(\d+)"',
 175             webpage, 'season number', default=None))
 176
 177         episode_number = int_or_none(self._search_regex(
 178             r'[^>]+itemprop="episode"[^>]*>\s*<meta[^>]+itemprop="episodeNumber"[^>]+content="(\d+)',
 179             webpage, 'episode number', default=None))
 180
 181         description = self._og_search_description(webpage, default=None) or self._html_search_meta(
 182             'description', webpage, 'description', default=None)
 183
 184         return {
 185             'id': video_id,
 186             'title': title,
 187             'series': compilation,
 188             'season': season,
 189             'season_number': season_number,
 190             'episode': episode,
 191             'episode_number': episode_number,
 192             'thumbnails': thumbnails,
 193             'description': description,
 194             'duration': int_or_none(result.get('duration')),
 195             'formats': formats,
 196         }
 197
 198
 199 class IviCompilationIE(InfoExtractor):
 200     IE_DESC = 'ivi.ru compilations'
 201     IE_NAME = 'ivi:compilation'
 202     _VALID_URL = r'https?://(?:www\.)?ivi\.ru/watch/(?!\d+)(?P<compilationid>[a-z\d_-]+)(?:/season(?P<seasonid>\d+))?$'
 203     _TESTS = [{
 204         'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa',
 205         'info_dict': {
 206             'id': 'dvoe_iz_lartsa',
 207             'title': 'Двое из ларца (2006 - 2008)',
 208         },
 209         'playlist_mincount': 24,
 210     }, {
 211         'url': 'http://www.ivi.ru/watch/dvoe_iz_lartsa/season1',
 212         'info_dict': {
 213             'id': 'dvoe_iz_lartsa/season1',
 214             'title': 'Двое из ларца (2006 - 2008) 1 сезон',
 215         },
 216         'playlist_mincount': 12,
 217     }]
 218
 219     def _extract_entries(self, html, compilation_id):
 220         return [
 221             self.url_result(
 222                 f'http://www.ivi.ru/watch/{compilation_id}/{serie}', IviIE.ie_key())
 223             for serie in re.findall(
 224                 rf'<a\b[^>]+\bhref=["\']/watch/{compilation_id}/(\d+)["\']', html)]
 225
 226     def _real_extract(self, url):
 227         mobj = self._match_valid_url(url)
 228         compilation_id = mobj.group('compilationid')
 229         season_id = mobj.group('seasonid')
 230
 231         if season_id is not None:  # Season link
 232             season_page = self._download_webpage(
 233                 url, compilation_id, f'Downloading season {season_id} web page')
 234             playlist_id = f'{compilation_id}/season{season_id}'
 235             playlist_title = self._html_search_meta('title', season_page, 'title')
 236             entries = self._extract_entries(season_page, compilation_id)
 237         else:  # Compilation link
 238             compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page')
 239             playlist_id = compilation_id
 240             playlist_title = self._html_search_meta('title', compilation_page, 'title')
 241             seasons = re.findall(
 242                 rf'<a href="/watch/{compilation_id}/season(\d+)', compilation_page)
 243             if not seasons:  # No seasons in this compilation
 244                 entries = self._extract_entries(compilation_page, compilation_id)
 245             else:
 246                 entries = []
 247                 for season_id in seasons:
 248                     season_page = self._download_webpage(
 249                         f'http://www.ivi.ru/watch/{compilation_id}/season{season_id}',
 250                         compilation_id, f'Downloading season {season_id} web page')
 251                     entries.extend(self._extract_entries(season_page, compilation_id))
 252
 253         return self.playlist_result(entries, playlist_id, playlist_title)