yt_dlp/extractor/rcti.py

   1 import json
   2 import random
   3 import time
   4
   5 from .common import InfoExtractor
   6 from ..networking.exceptions import HTTPError
   7 from ..utils import (
   8     ExtractorError,
   9     dict_get,
  10     strip_or_none,
  11     traverse_obj,
  12     try_get,
  13 )
  14
  15
  16 class RCTIPlusBaseIE(InfoExtractor):
  17     def _real_initialize(self):
  18         self._AUTH_KEY = self._download_json(
  19             'https://api.rctiplus.com/api/v1/visitor?platform=web',  # platform can be web, mweb, android, ios
  20             None, 'Fetching authorization key')['data']['access_token']
  21
  22     def _call_api(self, url, video_id, note=None):
  23         json = self._download_json(
  24             url, video_id, note=note, headers={'Authorization': self._AUTH_KEY})
  25         if json.get('status', {}).get('code', 0) != 0:
  26             raise ExtractorError(f'{self.IE_NAME} said: {json["status"]["message_client"]}', cause=json)
  27         return json.get('data'), json.get('meta')
  28
  29
  30 class RCTIPlusIE(RCTIPlusBaseIE):
  31     _VALID_URL = r'https?://www\.rctiplus\.com/(?:programs/\d+?/.*?/)?(?P<type>episode|clip|extra|live-event|missed-event)/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
  32     _TESTS = [{
  33         'url': 'https://www.rctiplus.com/programs/1259/kiko-untuk-lola/episode/22124/untuk-lola',
  34         'md5': '56ed45affad45fa18d5592a1bc199997',
  35         'info_dict': {
  36             'id': 'v_e22124',
  37             'title': 'Untuk Lola',
  38             'display_id': 'untuk-lola',
  39             'description': 'md5:2b809075c0b1e071e228ad6d13e41deb',
  40             'ext': 'mp4',
  41             'duration': 1400,
  42             'timestamp': 1615978800,
  43             'upload_date': '20210317',
  44             'series': 'Kiko : Untuk Lola',
  45             'season_number': 1,
  46             'episode_number': 1,
  47             'channel': 'RCTI',
  48         },
  49         'params': {
  50             'fixup': 'never',
  51         },
  52     }, {  # Clip; Series title doesn't appear on metadata JSON
  53         'url': 'https://www.rctiplus.com/programs/316/cahaya-terindah/clip/3921/make-a-wish',
  54         'md5': 'd179b2ff356f0e91a53bcc6a4d8504f0',
  55         'info_dict': {
  56             'id': 'v_c3921',
  57             'title': 'Make A Wish',
  58             'display_id': 'make-a-wish',
  59             'description': 'Make A Wish',
  60             'ext': 'mp4',
  61             'duration': 288,
  62             'timestamp': 1571652600,
  63             'upload_date': '20191021',
  64             'series': 'Cahaya Terindah',
  65             'channel': 'RCTI',
  66         },
  67         'params': {
  68             'fixup': 'never',
  69         },
  70     }, {  # Extra
  71         'url': 'https://www.rctiplus.com/programs/616/inews-malam/extra/9438/diungkapkan-melalui-surat-terbuka-ceo-ruangguru-belva-devara-mundur-dari-staf-khusus-presiden',
  72         'md5': 'c48106afdbce609749f5e0c007d9278a',
  73         'info_dict': {
  74             'id': 'v_ex9438',
  75             'title': 'md5:2ede828c0f8bde249e0912be150314ca',
  76             'display_id': 'md5:62b8d4e9ff096db527a1ad797e8a9933',
  77             'description': 'md5:2ede828c0f8bde249e0912be150314ca',
  78             'ext': 'mp4',
  79             'duration': 93,
  80             'timestamp': 1587561540,
  81             'upload_date': '20200422',
  82             'series': 'iNews Malam',
  83             'channel': 'INews',
  84         },
  85     }, {  # Missed event/replay
  86         'url': 'https://www.rctiplus.com/missed-event/2507/mou-signing-ceremony-27-juli-2021-1400-wib',
  87         'md5': '649c5f27250faed1452ca8b91e06922d',
  88         'info_dict': {
  89             'id': 'v_pe2507',
  90             'title': 'MOU Signing Ceremony | 27 Juli 2021 | 14.00 WIB',
  91             'display_id': 'mou-signing-ceremony-27-juli-2021-1400-wib',
  92             'ext': 'mp4',
  93             'timestamp': 1627142400,
  94             'upload_date': '20210724',
  95             'was_live': True,
  96             'release_timestamp': 1627369200,
  97         },
  98         'params': {
  99             'fixup': 'never',
 100         },
 101     }, {  # Live event; Cloudfront CDN
 102         'url': 'https://www.rctiplus.com/live-event/2530/dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
 103         'info_dict': {
 104             'id': 'v_le2530',
 105             'title': 'Dai Muda : Charging Imun dengan Iman | 4 Agustus 2021 | 16.00 WIB',
 106             'display_id': 'dai-muda-charging-imun-dengan-iman-4-agustus-2021-1600-wib',
 107             'ext': 'mp4',
 108             'timestamp': 1627898400,
 109             'upload_date': '20210802',
 110             'release_timestamp': 1628067600,
 111         },
 112         'params': {
 113             'skip_download': True,
 114         },
 115         'skip': 'This live event has ended.',
 116     }, {  # TV; live_at is null
 117         'url': 'https://www.rctiplus.com/live-event/1/rcti',
 118         'info_dict': {
 119             'id': 'v_lt1',
 120             'title': 'RCTI',
 121             'display_id': 'rcti',
 122             'ext': 'mp4',
 123             'timestamp': 1546344000,
 124             'upload_date': '20190101',
 125             'is_live': True,
 126         },
 127         'params': {
 128             'skip_download': True,
 129         },
 130     }]
 131     _CONVIVA_JSON_TEMPLATE = {
 132         't': 'CwsSessionHb',
 133         'cid': 'ff84ae928c3b33064b76dec08f12500465e59a6f',
 134         'clid': '0',
 135         'sid': 0,
 136         'seq': 0,
 137         'caps': 0,
 138         'sf': 7,
 139         'sdk': True,
 140     }
 141
 142     def _real_extract(self, url):
 143         match = self._match_valid_url(url).groupdict()
 144         video_type, video_id, display_id = match['type'], match['id'], match['display_id']
 145
 146         url_api_version = 'v2' if video_type == 'missed-event' else 'v1'
 147         appier_id = '23984824_' + str(random.randint(0, 10000000000))  # Based on the webpage's uuidRandom generator
 148         video_json = self._call_api(
 149             f'https://api.rctiplus.com/api/{url_api_version}/{video_type}/{video_id}/url?appierid={appier_id}', display_id, 'Downloading video URL JSON')[0]
 150         video_url = video_json['url']
 151
 152         is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['live_at'])
 153         if is_upcoming is None:
 154             is_upcoming = try_get(video_json, lambda x: x['current_date'] < x['start_date'])
 155         if is_upcoming:
 156             self.raise_no_formats(
 157                 'This event will start at {}.'.format(video_json['live_label']) if video_json.get('live_label') else 'This event has not started yet.', expected=True)
 158         if 'akamaized' in video_url:
 159             # For some videos hosted on Akamai's CDN (possibly AES-encrypted ones?), a session needs to at least be made via Conviva's API
 160             conviva_json_data = {
 161                 **self._CONVIVA_JSON_TEMPLATE,
 162                 'url': video_url,
 163                 'sst': int(time.time()),
 164             }
 165             conviva_json_res = self._download_json(
 166                 'https://ff84ae928c3b33064b76dec08f12500465e59a6f.cws.conviva.com/0/wsg', display_id,
 167                 'Creating Conviva session', 'Failed to create Conviva session',
 168                 fatal=False, data=json.dumps(conviva_json_data).encode())
 169             if conviva_json_res and conviva_json_res.get('err') != 'ok':
 170                 self.report_warning('Conviva said: {}'.format(str(conviva_json_res.get('err'))))
 171
 172         video_meta, meta_paths = self._call_api(
 173             f'https://api.rctiplus.com/api/v1/{video_type}/{video_id}', display_id, 'Downloading video metadata')
 174
 175         thumbnails, image_path = [], meta_paths.get('image_path', 'https://rstatic.akamaized.net/media/')
 176         if video_meta.get('portrait_image'):
 177             thumbnails.append({
 178                 'id': 'portrait_image',
 179                 'url': '{}{}{}'.format(image_path, 2000, video_meta['portrait_image']),  # 2000px seems to be the highest resolution that can be given
 180             })
 181         if video_meta.get('landscape_image'):
 182             thumbnails.append({
 183                 'id': 'landscape_image',
 184                 'url': '{}{}{}'.format(image_path, 2000, video_meta['landscape_image']),
 185             })
 186         try:
 187             formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
 188         except ExtractorError as e:
 189             if isinstance(e.cause, HTTPError) and e.cause.status == 403:
 190                 self.raise_geo_restricted(countries=['ID'], metadata_available=True)
 191             else:
 192                 raise e
 193         for f in formats:
 194             if 'akamaized' in f['url'] or 'cloudfront' in f['url']:
 195                 f.setdefault('http_headers', {})['Referer'] = 'https://www.rctiplus.com/'  # Referer header is required for akamai/cloudfront CDNs
 196
 197         return {
 198             'id': video_meta.get('product_id') or video_json.get('product_id'),
 199             'title': dict_get(video_meta, ('title', 'name')) or dict_get(video_json, ('content_name', 'assets_name')),
 200             'display_id': display_id,
 201             'description': video_meta.get('summary'),
 202             'timestamp': video_meta.get('release_date') or video_json.get('start_date'),
 203             'duration': video_meta.get('duration'),
 204             'categories': [video_meta['genre']] if video_meta.get('genre') else None,
 205             'average_rating': video_meta.get('star_rating'),
 206             'series': video_meta.get('program_title') or video_json.get('program_title'),
 207             'season_number': video_meta.get('season'),
 208             'episode_number': video_meta.get('episode'),
 209             'channel': video_json.get('tv_name'),
 210             'channel_id': video_json.get('tv_id'),
 211             'formats': formats,
 212             'thumbnails': thumbnails,
 213             'is_live': video_type == 'live-event' and not is_upcoming,
 214             'was_live': video_type == 'missed-event',
 215             'live_status': 'is_upcoming' if is_upcoming else None,
 216             'release_timestamp': video_json.get('live_at'),
 217         }
 218
 219
 220 class RCTIPlusSeriesIE(RCTIPlusBaseIE):
 221     _VALID_URL = r'https?://www\.rctiplus\.com/programs/(?P<id>\d+)/(?P<display_id>[^/?#&]+)(?:/(?P<type>episodes|extras|clips))?'
 222     _TESTS = [{
 223         'url': 'https://www.rctiplus.com/programs/829/putri-untuk-pangeran',
 224         'playlist_mincount': 1019,
 225         'info_dict': {
 226             'id': '829',
 227             'title': 'Putri Untuk Pangeran',
 228             'description': 'md5:aca7b54d05bd95a67d4f4613cc1d622d',
 229             'age_limit': 2,
 230             'cast': ['Verrel Bramasta', 'Ranty Maria', 'Riza Syah', 'Ivan Fadilla', 'Nicole Parham', 'Dll', 'Aviv Elham'],
 231             'display_id': 'putri-untuk-pangeran',
 232             'tags': 'count:18',
 233         },
 234     }, {  # No episodes
 235         'url': 'https://www.rctiplus.com/programs/615/inews-pagi',
 236         'playlist_mincount': 388,
 237         'info_dict': {
 238             'id': '615',
 239             'title': 'iNews Pagi',
 240             'description': 'md5:f18ee3d4643cfb41c358e5a9b693ee04',
 241             'age_limit': 2,
 242             'tags': 'count:11',
 243             'display_id': 'inews-pagi',
 244         },
 245     }]
 246     _AGE_RATINGS = {  # Based off https://id.wikipedia.org/wiki/Sistem_rating_konten_televisi with additional ratings
 247         'S-SU': 2,
 248         'SU': 2,
 249         'P': 2,
 250         'A': 7,
 251         'R': 13,
 252         'R-R/1': 17,  # Labelled as 17+ despite being R
 253         'D': 18,
 254     }
 255
 256     @classmethod
 257     def suitable(cls, url):
 258         return False if RCTIPlusIE.suitable(url) else super().suitable(url)
 259
 260     def _entries(self, url, display_id=None, note='Downloading entries JSON', metadata={}):
 261         total_pages = 0
 262         try:
 263             total_pages = self._call_api(
 264                 f'{url}&length=20&page=0',
 265                 display_id, note)[1]['pagination']['total_page']
 266         except ExtractorError as e:
 267             if 'not found' in str(e):
 268                 return []
 269             raise e
 270         if total_pages <= 0:
 271             return []
 272
 273         for page_num in range(1, total_pages + 1):
 274             episode_list = self._call_api(
 275                 f'{url}&length=20&page={page_num}',
 276                 display_id, f'{note} page {page_num}')[0] or []
 277
 278             for video_json in episode_list:
 279                 yield {
 280                     '_type': 'url',
 281                     'url': video_json['share_link'],
 282                     'ie_key': RCTIPlusIE.ie_key(),
 283                     'id': video_json.get('product_id'),
 284                     'title': video_json.get('title'),
 285                     'display_id': video_json.get('title_code').replace('_', '-'),
 286                     'description': video_json.get('summary'),
 287                     'timestamp': video_json.get('release_date'),
 288                     'duration': video_json.get('duration'),
 289                     'season_number': video_json.get('season'),
 290                     'episode_number': video_json.get('episode'),
 291                     **metadata,
 292                 }
 293
 294     def _series_entries(self, series_id, display_id=None, video_type=None, metadata={}):
 295         if not video_type or video_type in 'episodes':
 296             try:
 297                 seasons_list = self._call_api(
 298                     f'https://api.rctiplus.com/api/v1/program/{series_id}/season',
 299                     display_id, 'Downloading seasons list JSON')[0]
 300             except ExtractorError as e:
 301                 if 'not found' not in str(e):
 302                     raise
 303                 seasons_list = []
 304             for season in seasons_list:
 305                 yield from self._entries(
 306                     f'https://api.rctiplus.com/api/v2/program/{series_id}/episode?season={season["season"]}',
 307                     display_id, f'Downloading season {season["season"]} episode entries', metadata)
 308         if not video_type or video_type in 'extras':
 309             yield from self._entries(
 310                 f'https://api.rctiplus.com/api/v2/program/{series_id}/extra?content_id=0',
 311                 display_id, 'Downloading extra entries', metadata)
 312         if not video_type or video_type in 'clips':
 313             yield from self._entries(
 314                 f'https://api.rctiplus.com/api/v2/program/{series_id}/clip?content_id=0',
 315                 display_id, 'Downloading clip entries', metadata)
 316
 317     def _real_extract(self, url):
 318         series_id, display_id, video_type = self._match_valid_url(url).group('id', 'display_id', 'type')
 319         if video_type:
 320             self.report_warning(
 321                 f'Only {video_type} will be downloaded. '
 322                 f'To download everything from the series, remove "/{video_type}" from the URL')
 323
 324         series_meta, meta_paths = self._call_api(
 325             f'https://api.rctiplus.com/api/v1/program/{series_id}/detail', display_id, 'Downloading series metadata')
 326         metadata = {
 327             'age_limit': try_get(series_meta, lambda x: self._AGE_RATINGS[x['age_restriction'][0]['code']]),
 328             'cast': traverse_obj(series_meta, (('starring', 'creator', 'writer'), ..., 'name'),
 329                                  expected_type=lambda x: strip_or_none(x) or None),
 330             'tags': traverse_obj(series_meta, ('tag', ..., 'name'),
 331                                  expected_type=lambda x: strip_or_none(x) or None),
 332         }
 333         return self.playlist_result(
 334             self._series_entries(series_id, display_id, video_type, metadata), series_id,
 335             series_meta.get('title'), series_meta.get('summary'), display_id=display_id, **metadata)
 336
 337
 338 class RCTIPlusTVIE(RCTIPlusBaseIE):
 339     _VALID_URL = r'https?://www\.rctiplus\.com/((tv/(?P<tvname>\w+))|(?P<eventname>live-event|missed-event))'
 340     _TESTS = [{
 341         'url': 'https://www.rctiplus.com/tv/rcti',
 342         'info_dict': {
 343             'id': 'v_lt1',
 344             'title': 'RCTI',
 345             'ext': 'mp4',
 346             'timestamp': 1546344000,
 347             'upload_date': '20190101',
 348         },
 349         'params': {
 350             'skip_download': True,
 351         },
 352     }, {
 353         # Returned video will always change
 354         'url': 'https://www.rctiplus.com/live-event',
 355         'only_matching': True,
 356     }, {
 357         # Returned video will also always change
 358         'url': 'https://www.rctiplus.com/missed-event',
 359         'only_matching': True,
 360     }]
 361
 362     @classmethod
 363     def suitable(cls, url):
 364         return False if RCTIPlusIE.suitable(url) else super().suitable(url)
 365
 366     def _real_extract(self, url):
 367         match = self._match_valid_url(url).groupdict()
 368         tv_id = match.get('tvname') or match.get('eventname')
 369         webpage = self._download_webpage(url, tv_id)
 370         video_type, video_id = self._search_regex(
 371             r'url\s*:\s*["\']https://api\.rctiplus\.com/api/v./(?P<type>[^/]+)/(?P<id>\d+)/url',
 372             webpage, 'video link', group=('type', 'id'))
 373         return self.url_result(f'https://www.rctiplus.com/{video_type}/{video_id}/{tv_id}', 'RCTIPlus')