yt_dlp/extractor/crackle.py

   1 import hashlib
   2 import hmac
   3 import re
   4 import time
   5
   6 from .common import InfoExtractor
   7 from ..networking.exceptions import HTTPError
   8 from ..utils import (
   9     determine_ext,
  10     float_or_none,
  11     int_or_none,
  12     orderedSet,
  13     parse_age_limit,
  14     parse_duration,
  15     url_or_none,
  16     ExtractorError
  17 )
  18
  19
  20 class CrackleIE(InfoExtractor):
  21     _VALID_URL = r'(?:crackle:|https?://(?:(?:www|m)\.)?(?:sony)?crackle\.com/(?:playlist/\d+/|(?:[^/]+/)+))(?P<id>\d+)'
  22     _TESTS = [{
  23         # Crackle is available in the United States and territories
  24         'url': 'https://www.crackle.com/thanksgiving/2510064',
  25         'info_dict': {
  26             'id': '2510064',
  27             'ext': 'mp4',
  28             'title': 'Touch Football',
  29             'description': 'md5:cfbb513cf5de41e8b56d7ab756cff4df',
  30             'duration': 1398,
  31             'view_count': int,
  32             'average_rating': 0,
  33             'age_limit': 17,
  34             'genre': 'Comedy',
  35             'creator': 'Daniel Powell',
  36             'artist': 'Chris Elliott, Amy Sedaris',
  37             'release_year': 2016,
  38             'series': 'Thanksgiving',
  39             'episode': 'Touch Football',
  40             'season_number': 1,
  41             'episode_number': 1,
  42         },
  43         'params': {
  44             # m3u8 download
  45             'skip_download': True,
  46         },
  47         'expected_warnings': [
  48             'Trying with a list of known countries'
  49         ],
  50     }, {
  51         'url': 'https://www.sonycrackle.com/thanksgiving/2510064',
  52         'only_matching': True,
  53     }]
  54
  55     _MEDIA_FILE_SLOTS = {
  56         '360p.mp4': {
  57             'width': 640,
  58             'height': 360,
  59         },
  60         '480p.mp4': {
  61             'width': 768,
  62             'height': 432,
  63         },
  64         '480p_1mbps.mp4': {
  65             'width': 852,
  66             'height': 480,
  67         },
  68     }
  69
  70     def _download_json(self, url, *args, **kwargs):
  71         # Authorization generation algorithm is reverse engineered from:
  72         # https://www.sonycrackle.com/static/js/main.ea93451f.chunk.js
  73         timestamp = time.strftime('%Y%m%d%H%M', time.gmtime())
  74         h = hmac.new(b'IGSLUQCBDFHEOIFM', '|'.join([url, timestamp]).encode(), hashlib.sha1).hexdigest().upper()
  75         headers = {
  76             'Accept': 'application/json',
  77             'Authorization': '|'.join([h, timestamp, '117', '1']),
  78         }
  79         return InfoExtractor._download_json(self, url, *args, headers=headers, **kwargs)
  80
  81     def _real_extract(self, url):
  82         video_id = self._match_id(url)
  83
  84         geo_bypass_country = self.get_param('geo_bypass_country', None)
  85         countries = orderedSet((geo_bypass_country, 'US', 'AU', 'CA', 'AS', 'FM', 'GU', 'MP', 'PR', 'PW', 'MH', 'VI', ''))
  86         num_countries, num = len(countries) - 1, 0
  87
  88         media = {}
  89         for num, country in enumerate(countries):
  90             if num == 1:  # start hard-coded list
  91                 self.report_warning('%s. Trying with a list of known countries' % (
  92                     'Unable to obtain video formats from %s API' % geo_bypass_country if geo_bypass_country
  93                     else 'No country code was given using --geo-bypass-country'))
  94             elif num == num_countries:  # end of list
  95                 geo_info = self._download_json(
  96                     'https://web-api-us.crackle.com/Service.svc/geo/country',
  97                     video_id, fatal=False, note='Downloading geo-location information from crackle API',
  98                     errnote='Unable to fetch geo-location information from crackle') or {}
  99                 country = geo_info.get('CountryCode')
 100                 if country is None:
 101                     continue
 102                 self.to_screen('%s identified country as %s' % (self.IE_NAME, country))
 103                 if country in countries:
 104                     self.to_screen('Downloading from %s API was already attempted. Skipping...' % country)
 105                     continue
 106
 107             if country is None:
 108                 continue
 109             try:
 110                 media = self._download_json(
 111                     'https://web-api-us.crackle.com/Service.svc/details/media/%s/%s?disableProtocols=true' % (video_id, country),
 112                     video_id, note='Downloading media JSON from %s API' % country,
 113                     errnote='Unable to download media JSON')
 114             except ExtractorError as e:
 115                 # 401 means geo restriction, trying next country
 116                 if isinstance(e.cause, HTTPError) and e.cause.status == 401:
 117                     continue
 118                 raise
 119
 120             status = media.get('status')
 121             if status.get('messageCode') != '0':
 122                 raise ExtractorError(
 123                     '%s said: %s %s - %s' % (
 124                         self.IE_NAME, status.get('messageCodeDescription'), status.get('messageCode'), status.get('message')),
 125                     expected=True)
 126
 127             # Found video formats
 128             if isinstance(media.get('MediaURLs'), list):
 129                 break
 130
 131         ignore_no_formats = self.get_param('ignore_no_formats_error')
 132
 133         if not media or (not media.get('MediaURLs') and not ignore_no_formats):
 134             raise ExtractorError(
 135                 'Unable to access the crackle API. Try passing your country code '
 136                 'to --geo-bypass-country. If it still does not work and the '
 137                 'video is available in your country')
 138         title = media['Title']
 139
 140         formats, subtitles = [], {}
 141         has_drm = False
 142         for e in media.get('MediaURLs') or []:
 143             if e.get('UseDRM'):
 144                 has_drm = True
 145                 format_url = url_or_none(e.get('DRMPath'))
 146             else:
 147                 format_url = url_or_none(e.get('Path'))
 148             if not format_url:
 149                 continue
 150             ext = determine_ext(format_url)
 151             if ext == 'm3u8':
 152                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
 153                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
 154                     m3u8_id='hls', fatal=False)
 155                 formats.extend(fmts)
 156                 subtitles = self._merge_subtitles(subtitles, subs)
 157             elif ext == 'mpd':
 158                 fmts, subs = self._extract_mpd_formats_and_subtitles(
 159                     format_url, video_id, mpd_id='dash', fatal=False)
 160                 formats.extend(fmts)
 161                 subtitles = self._merge_subtitles(subtitles, subs)
 162             elif format_url.endswith('.ism/Manifest'):
 163                 fmts, subs = self._extract_ism_formats_and_subtitles(
 164                     format_url, video_id, ism_id='mss', fatal=False)
 165                 formats.extend(fmts)
 166                 subtitles = self._merge_subtitles(subtitles, subs)
 167             else:
 168                 mfs_path = e.get('Type')
 169                 mfs_info = self._MEDIA_FILE_SLOTS.get(mfs_path)
 170                 if not mfs_info:
 171                     continue
 172                 formats.append({
 173                     'url': format_url,
 174                     'format_id': 'http-' + mfs_path.split('.')[0],
 175                     'width': mfs_info['width'],
 176                     'height': mfs_info['height'],
 177                 })
 178         if not formats and has_drm:
 179             self.report_drm(video_id)
 180
 181         description = media.get('Description')
 182         duration = int_or_none(media.get(
 183             'DurationInSeconds')) or parse_duration(media.get('Duration'))
 184         view_count = int_or_none(media.get('CountViews'))
 185         average_rating = float_or_none(media.get('UserRating'))
 186         age_limit = parse_age_limit(media.get('Rating'))
 187         genre = media.get('Genre')
 188         release_year = int_or_none(media.get('ReleaseYear'))
 189         creator = media.get('Directors')
 190         artist = media.get('Cast')
 191
 192         if media.get('MediaTypeDisplayValue') == 'Full Episode':
 193             series = media.get('ShowName')
 194             episode = title
 195             season_number = int_or_none(media.get('Season'))
 196             episode_number = int_or_none(media.get('Episode'))
 197         else:
 198             series = episode = season_number = episode_number = None
 199
 200         cc_files = media.get('ClosedCaptionFiles')
 201         if isinstance(cc_files, list):
 202             for cc_file in cc_files:
 203                 if not isinstance(cc_file, dict):
 204                     continue
 205                 cc_url = url_or_none(cc_file.get('Path'))
 206                 if not cc_url:
 207                     continue
 208                 lang = cc_file.get('Locale') or 'en'
 209                 subtitles.setdefault(lang, []).append({'url': cc_url})
 210
 211         thumbnails = []
 212         images = media.get('Images')
 213         if isinstance(images, list):
 214             for image_key, image_url in images.items():
 215                 mobj = re.search(r'Img_(\d+)[xX](\d+)', image_key)
 216                 if not mobj:
 217                     continue
 218                 thumbnails.append({
 219                     'url': image_url,
 220                     'width': int(mobj.group(1)),
 221                     'height': int(mobj.group(2)),
 222                 })
 223
 224         return {
 225             'id': video_id,
 226             'title': title,
 227             'description': description,
 228             'duration': duration,
 229             'view_count': view_count,
 230             'average_rating': average_rating,
 231             'age_limit': age_limit,
 232             'genre': genre,
 233             'creator': creator,
 234             'artist': artist,
 235             'release_year': release_year,
 236             'series': series,
 237             'episode': episode,
 238             'season_number': season_number,
 239             'episode_number': episode_number,
 240             'thumbnails': thumbnails,
 241             'subtitles': subtitles,
 242             'formats': formats,
 243         }