yt_dlp/extractor/go.py

   1 import re
   2
   3 from .adobepass import AdobePassIE
   4 from ..utils import (
   5     ExtractorError,
   6     determine_ext,
   7     int_or_none,
   8     join_nonempty,
   9     parse_age_limit,
  10     remove_end,
  11     remove_start,
  12     traverse_obj,
  13     try_get,
  14     unified_timestamp,
  15     urlencode_postdata,
  16 )
  17
  18
  19 class GoIE(AdobePassIE):
  20     _SITE_INFO = {
  21         'abc': {
  22             'brand': '001',
  23             'requestor_id': 'ABC',
  24         },
  25         'freeform': {
  26             'brand': '002',
  27             'requestor_id': 'ABCFamily',
  28         },
  29         'watchdisneychannel': {
  30             'brand': '004',
  31             'resource_id': 'Disney',
  32         },
  33         'watchdisneyjunior': {
  34             'brand': '008',
  35             'resource_id': 'DisneyJunior',
  36         },
  37         'watchdisneyxd': {
  38             'brand': '009',
  39             'resource_id': 'DisneyXD',
  40         },
  41         'disneynow': {
  42             'brand': '011',
  43             'resource_id': 'Disney',
  44         },
  45         'fxnow.fxnetworks': {
  46             'brand': '025',
  47             'requestor_id': 'dtci',
  48         },
  49     }
  50     _VALID_URL = r'''(?x)
  51                     https?://
  52                         (?P<sub_domain>
  53                             (?:{}\.)?go|fxnow\.fxnetworks|
  54                             (?:www\.)?(?:abc|freeform|disneynow)
  55                         )\.com/
  56                         (?:
  57                             (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
  58                             (?:[^/]+/)*(?P<display_id>[^/?\#]+)
  59                         )
  60                     '''.format(r'\.|'.join(list(_SITE_INFO.keys())))
  61     _TESTS = [{
  62         'url': 'http://abc.go.com/shows/designated-survivor/video/most-recent/VDKA3807643',
  63         'info_dict': {
  64             'id': 'VDKA3807643',
  65             'ext': 'mp4',
  66             'title': 'The Traitor in the White House',
  67             'description': 'md5:05b009d2d145a1e85d25111bd37222e8',
  68         },
  69         'params': {
  70             # m3u8 download
  71             'skip_download': True,
  72         },
  73         'skip': 'This content is no longer available.',
  74     }, {
  75         'url': 'https://disneynow.com/shows/big-hero-6-the-series',
  76         'info_dict': {
  77             'title': 'Doraemon',
  78             'id': 'SH55574025',
  79         },
  80         'playlist_mincount': 51,
  81     }, {
  82         'url': 'http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood',
  83         'info_dict': {
  84             'id': 'VDKA3609139',
  85             'title': 'This Guilty Blood',
  86             'description': 'md5:f18e79ad1c613798d95fdabfe96cd292',
  87             'age_limit': 14,
  88             'episode': 'Episode 1',
  89             'upload_date': '20170102',
  90             'season': 'Season 2',
  91             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abcf/Shadowhunters/video/201/ae5f75608d86bf88aa4f9f4aa76ab1b7/579x325-Q100_ae5f75608d86bf88aa4f9f4aa76ab1b7.jpg',
  92             'duration': 2544,
  93             'season_number': 2,
  94             'series': 'Shadowhunters',
  95             'episode_number': 1,
  96             'timestamp': 1483387200,
  97             'ext': 'mp4',
  98         },
  99         'params': {
 100             'geo_bypass_ip_block': '3.244.239.0/24',
 101             # m3u8 download
 102             'skip_download': True,
 103         },
 104     }, {
 105         'url': 'https://abc.com/shows/the-rookie/episode-guide/season-04/12-the-knock',
 106         'info_dict': {
 107             'id': 'VDKA26050359',
 108             'title': 'The Knock',
 109             'description': 'md5:0c2947e3ada4c31f28296db7db14aa64',
 110             'age_limit': 14,
 111             'ext': 'mp4',
 112             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/abc/TheRookie/video/412/daf830d06e83b11eaf5c0a299d993ae3/1556x876-Q75_daf830d06e83b11eaf5c0a299d993ae3.jpg',
 113             'episode': 'Episode 12',
 114             'season_number': 4,
 115             'season': 'Season 4',
 116             'timestamp': 1642975200,
 117             'episode_number': 12,
 118             'upload_date': '20220123',
 119             'series': 'The Rookie',
 120             'duration': 2572,
 121         },
 122         'params': {
 123             'geo_bypass_ip_block': '3.244.239.0/24',
 124             # m3u8 download
 125             'skip_download': True,
 126         },
 127     }, {
 128         'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
 129         'info_dict': {
 130             'id': 'VDKA12782841',
 131             'title': 'First Look: Better Things - Season 2',
 132             'description': 'md5:fa73584a95761c605d9d54904e35b407',
 133             'ext': 'mp4',
 134             'age_limit': 14,
 135             'upload_date': '20170825',
 136             'duration': 161,
 137             'series': 'Better Things',
 138             'thumbnail': 'http://cdn1.edgedatg.com/aws/v2/fx/BetterThings/video/12782841/b6b05e58264121cc2c98811318e6d507/1556x876-Q75_b6b05e58264121cc2c98811318e6d507.jpg',
 139             'timestamp': 1503661074,
 140         },
 141         'params': {
 142             'geo_bypass_ip_block': '3.244.239.0/24',
 143             # m3u8 download
 144             'skip_download': True,
 145         },
 146     }, {
 147         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
 148         'only_matching': True,
 149     }, {
 150         'url': 'http://abc.go.com/shows/world-news-tonight/episode-guide/2017-02/17-021717-intense-stand-off-between-man-with-rifle-and-police-in-oakland',
 151         'only_matching': True,
 152     }, {
 153         # brand 004
 154         'url': 'http://disneynow.go.com/shows/big-hero-6-the-series/season-01/episode-10-mr-sparkles-loses-his-sparkle/vdka4637915',
 155         'only_matching': True,
 156     }, {
 157         # brand 008
 158         'url': 'http://disneynow.go.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
 159         'only_matching': True,
 160     }, {
 161         'url': 'https://disneynow.com/shows/minnies-bow-toons/video/happy-campers/vdka4872013',
 162         'only_matching': True,
 163     }, {
 164         'url': 'https://www.freeform.com/shows/cruel-summer/episode-guide/season-01/01-happy-birthday-jeanette-turner',
 165         'only_matching': True,
 166     }]
 167
 168     def _extract_videos(self, brand, video_id='-1', show_id='-1'):
 169         display_id = video_id if video_id != '-1' else show_id
 170         return self._download_json(
 171             f'http://api.contents.watchabc.go.com/vp2/ws/contents/3000/videos/{brand}/001/-1/{show_id}/-1/{video_id}/-1/-1.json',
 172             display_id)['video']
 173
 174     def _real_extract(self, url):
 175         mobj = self._match_valid_url(url)
 176         sub_domain = remove_start(remove_end(mobj.group('sub_domain') or '', '.go'), 'www.')
 177         video_id, display_id = mobj.group('id', 'display_id')
 178         site_info = self._SITE_INFO.get(sub_domain, {})
 179         brand = site_info.get('brand')
 180         if not video_id or not site_info:
 181             webpage = self._download_webpage(url, display_id or video_id)
 182             data = self._parse_json(
 183                 self._search_regex(
 184                     r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
 185                     'data', default='{}'),
 186                 display_id or video_id, fatal=False)
 187             # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
 188             layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
 189             video_id = None
 190             if layout:
 191                 video_id = try_get(
 192                     layout,
 193                     (lambda x: x['videoid'], lambda x: x['video']['id']),
 194                     str)
 195             if not video_id:
 196                 video_id = self._search_regex(
 197                     (
 198                         # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
 199                         # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
 200                         r'data-video-id=["\']*(VDKA\w+)',
 201                         # page.analytics.videoIdCode
 202                         r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
 203                         # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
 204                         r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)',
 205                     ), webpage, 'video id', default=video_id)
 206             if not site_info:
 207                 brand = self._search_regex(
 208                     (r'data-brand=\s*["\']\s*(\d+)',
 209                      r'data-page-brand=\s*["\']\s*(\d+)'), webpage, 'brand',
 210                     default='004')
 211                 site_info = next(
 212                     si for _, si in self._SITE_INFO.items()
 213                     if si.get('brand') == brand)
 214             if not video_id:
 215                 # show extraction works for Disney, DisneyJunior and DisneyXD
 216                 # ABC and Freeform has different layout
 217                 show_id = self._search_regex(r'data-show-id=["\']*(SH\d+)', webpage, 'show id')
 218                 videos = self._extract_videos(brand, show_id=show_id)
 219                 show_title = self._search_regex(r'data-show-title="([^"]+)"', webpage, 'show title', fatal=False)
 220                 entries = []
 221                 for video in videos:
 222                     entries.append(self.url_result(
 223                         video['url'], 'Go', video.get('id'), video.get('title')))
 224                 entries.reverse()
 225                 return self.playlist_result(entries, show_id, show_title)
 226         video_data = self._extract_videos(brand, video_id)[0]
 227         video_id = video_data['id']
 228         title = video_data['title']
 229
 230         formats = []
 231         subtitles = {}
 232         for asset in video_data.get('assets', {}).get('asset', []):
 233             asset_url = asset.get('value')
 234             if not asset_url:
 235                 continue
 236             format_id = asset.get('format')
 237             ext = determine_ext(asset_url)
 238             if ext == 'm3u8':
 239                 video_type = video_data.get('type')
 240                 data = {
 241                     'video_id': video_data['id'],
 242                     'video_type': video_type,
 243                     'brand': brand,
 244                     'device': '001',
 245                 }
 246                 if video_data.get('accesslevel') == '1':
 247                     requestor_id = site_info.get('requestor_id', 'DisneyChannels')
 248                     resource = site_info.get('resource_id') or self._get_mvpd_resource(
 249                         requestor_id, title, video_id, None)
 250                     auth = self._extract_mvpd_auth(
 251                         url, video_id, requestor_id, resource)
 252                     data.update({
 253                         'token': auth,
 254                         'token_type': 'ap',
 255                         'adobe_requestor_id': requestor_id,
 256                     })
 257                 else:
 258                     self._initialize_geo_bypass({'countries': ['US']})
 259                 entitlement = self._download_json(
 260                     'https://api.entitlement.watchabc.go.com/vp2/ws-secure/entitlement/2020/authorize.json',
 261                     video_id, data=urlencode_postdata(data))
 262                 errors = entitlement.get('errors', {}).get('errors', [])
 263                 if errors:
 264                     for error in errors:
 265                         if error.get('code') == 1002:
 266                             self.raise_geo_restricted(
 267                                 error['message'], countries=['US'])
 268                     error_message = ', '.join([error['message'] for error in errors])
 269                     raise ExtractorError(f'{self.IE_NAME} said: {error_message}', expected=True)
 270                 asset_url += '?' + entitlement['uplynkData']['sessionKey']
 271                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
 272                     asset_url, video_id, 'mp4', m3u8_id=format_id or 'hls', fatal=False)
 273                 formats.extend(fmts)
 274                 self._merge_subtitles(subs, target=subtitles)
 275             else:
 276                 f = {
 277                     'format_id': format_id,
 278                     'url': asset_url,
 279                     'ext': ext,
 280                 }
 281                 if re.search(r'(?:/mp4/source/|_source\.mp4)', asset_url):
 282                     f.update({
 283                         'format_id': (f'{format_id}-' if format_id else '') + 'SOURCE',
 284                         'quality': 1,
 285                     })
 286                 else:
 287                     mobj = re.search(r'/(\d+)x(\d+)/', asset_url)
 288                     if mobj:
 289                         height = int(mobj.group(2))
 290                         f.update({
 291                             'format_id': join_nonempty(format_id, f'{height}P'),
 292                             'width': int(mobj.group(1)),
 293                             'height': height,
 294                         })
 295                 formats.append(f)
 296
 297         for cc in video_data.get('closedcaption', {}).get('src', []):
 298             cc_url = cc.get('value')
 299             if not cc_url:
 300                 continue
 301             ext = determine_ext(cc_url)
 302             if ext == 'xml':
 303                 ext = 'ttml'
 304             subtitles.setdefault(cc.get('lang'), []).append({
 305                 'url': cc_url,
 306                 'ext': ext,
 307             })
 308
 309         thumbnails = []
 310         for thumbnail in video_data.get('thumbnails', {}).get('thumbnail', []):
 311             thumbnail_url = thumbnail.get('value')
 312             if not thumbnail_url:
 313                 continue
 314             thumbnails.append({
 315                 'url': thumbnail_url,
 316                 'width': int_or_none(thumbnail.get('width')),
 317                 'height': int_or_none(thumbnail.get('height')),
 318             })
 319
 320         return {
 321             'id': video_id,
 322             'title': title,
 323             'description': video_data.get('longdescription') or video_data.get('description'),
 324             'duration': int_or_none(video_data.get('duration', {}).get('value'), 1000),
 325             'age_limit': parse_age_limit(video_data.get('tvrating', {}).get('rating')),
 326             'episode_number': int_or_none(video_data.get('episodenumber')),
 327             'series': video_data.get('show', {}).get('title'),
 328             'season_number': int_or_none(video_data.get('season', {}).get('num')),
 329             'thumbnails': thumbnails,
 330             'formats': formats,
 331             'subtitles': subtitles,
 332             'timestamp': unified_timestamp(traverse_obj(video_data, ('airdates', 'airdate', 0))),
 333         }