yt_dlp/extractor/piksel.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     ExtractorError,
   6     dict_get,
   7     int_or_none,
   8     join_nonempty,
   9     parse_iso8601,
  10     traverse_obj,
  11     try_get,
  12     unescapeHTML,
  13     urljoin,
  14 )
  15
  16
  17 class PikselIE(InfoExtractor):
  18     _VALID_URL = r'''(?x)https?://
  19         (?:
  20             (?:
  21                 player\.
  22                     (?:
  23                         olympusattelecom|
  24                         vibebyvista
  25                     )|
  26                 (?:api|player)\.multicastmedia|
  27                 (?:api-ovp|player)\.piksel
  28             )\.(?:com|tech)|
  29             (?:
  30                 mz-edge\.stream\.co|
  31                 movie-s\.nhk\.or
  32             )\.jp|
  33             vidego\.baltimorecity\.gov
  34         )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
  35     _EMBED_REGEX = [r'<iframe[^>]+src=["\'](?P<url>(?:https?:)?//player\.piksel\.(?:com|tech)/v/[a-z0-9]+)']
  36     _TESTS = [
  37         {
  38             'url': 'http://player.piksel.tech/v/ums2867l',
  39             'md5': '34e34c8d89dc2559976a6079db531e85',
  40             'info_dict': {
  41                 'id': 'ums2867l',
  42                 'ext': 'mp4',
  43                 'title': 'GX-005 with Caption',
  44                 'timestamp': 1481335659,
  45                 'upload_date': '20161210',
  46                 'description': '',
  47                 'thumbnail': 'https://thumbs.piksel.tech/thumbs/aid/t1488331553/3238987.jpg?w=640&h=480',
  48             },
  49         },
  50         {
  51             # Original source: http://www.uscourts.gov/cameras-courts/state-washington-vs-donald-j-trump-et-al
  52             'url': 'https://player.piksel.tech/v/v80kqp41',
  53             'md5': '753ddcd8cc8e4fa2dda4b7be0e77744d',
  54             'info_dict': {
  55                 'id': 'v80kqp41',
  56                 'ext': 'mp4',
  57                 'title': 'WAW- State of Washington vs. Donald J. Trump, et al',
  58                 'description': 'State of Washington vs. Donald J. Trump, et al, Case Number 17-CV-00141-JLR, TRO Hearing, Civil Rights Case, 02/3/2017, 1:00 PM (PST), Seattle Federal Courthouse, Seattle, WA, Judge James L. Robart presiding.',
  59                 'timestamp': 1486171129,
  60                 'upload_date': '20170204',
  61                 'thumbnail': 'https://thumbs.piksel.tech/thumbs/aid/t1495569155/3279887.jpg?w=640&h=360',
  62             },
  63         },
  64         {
  65             # https://www3.nhk.or.jp/nhkworld/en/ondemand/video/2019240/
  66             'url': 'http://player.piksel.com/v/refid/nhkworld/prefid/nw_vod_v_en_2019_240_20190823233000_02_1566873477',
  67             'only_matching': True,
  68         },
  69     ]
  70
  71     def _call_api(self, app_token, resource, display_id, query, host='https://player.piksel.tech', fatal=True):
  72         url = urljoin(host, f'/ws/ws_{resource}/api/{app_token}/mode/json/apiv/5')
  73         response = traverse_obj(
  74             self._download_json(url, display_id, query=query, fatal=fatal), ('response', {dict})) or {}
  75         failure = traverse_obj(response, ('failure', 'reason')) if response else 'Empty response from API'
  76         if failure:
  77             if fatal:
  78                 raise ExtractorError(failure, expected=True)
  79             self.report_warning(failure)
  80         return response
  81
  82     def _real_extract(self, url):
  83         ref_id, display_id = self._match_valid_url(url).groups()
  84         webpage = self._download_webpage(url, display_id)
  85         app_token = self._search_regex([
  86             r'clientAPI\s*:\s*"([^"]+)"',
  87             r'data-de-api-key\s*=\s*"([^"]+)"',
  88         ], webpage, 'app token')
  89         query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
  90         program = self._call_api(
  91             app_token, 'program', display_id, query, url)['WsProgramResponse']['program']
  92         video_id = program['uuid']
  93         video_data = program['asset']
  94         title = video_data['title']
  95         asset_type = dict_get(video_data, ['assetType', 'asset_type'])
  96
  97         formats = []
  98
  99         def process_asset_file(asset_file):
 100             if not asset_file:
 101                 return
 102             # TODO: extract rtmp formats
 103             http_url = asset_file.get('http_url')
 104             if not http_url:
 105                 return
 106             tbr = None
 107             vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
 108             abr = int_or_none(asset_file.get('audioBitrate'), 1024)
 109             if asset_type == 'video':
 110                 tbr = vbr + abr
 111             elif asset_type == 'audio':
 112                 tbr = abr
 113
 114             formats.append({
 115                 'format_id': join_nonempty('http', tbr),
 116                 'url': unescapeHTML(http_url),
 117                 'vbr': vbr,
 118                 'abr': abr,
 119                 'width': int_or_none(asset_file.get('videoWidth')),
 120                 'height': int_or_none(asset_file.get('videoHeight')),
 121                 'filesize': int_or_none(asset_file.get('filesize')),
 122                 'tbr': tbr,
 123             })
 124
 125         def process_asset_files(asset_files):
 126             for asset_file in (asset_files or []):
 127                 process_asset_file(asset_file)
 128
 129         process_asset_files(video_data.get('assetFiles'))
 130         process_asset_file(video_data.get('referenceFile'))
 131         if not formats:
 132             asset_id = video_data.get('assetid') or program.get('assetid')
 133             if asset_id:
 134                 process_asset_files(try_get(self._call_api(
 135                     app_token, 'asset_file', display_id, {
 136                         'assetid': asset_id,
 137                     }, url, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
 138
 139         m3u8_url = dict_get(video_data, [
 140             'm3u8iPadURL',
 141             'ipadM3u8Url',
 142             'm3u8AndroidURL',
 143             'm3u8iPhoneURL',
 144             'iphoneM3u8Url'])
 145         if m3u8_url:
 146             formats.extend(self._extract_m3u8_formats(
 147                 m3u8_url, video_id, 'mp4', 'm3u8_native',
 148                 m3u8_id='hls', fatal=False))
 149
 150         smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
 151         if smil_url:
 152             transform_source = lambda x: x.replace('src="/', 'src="')
 153             if ref_id == 'nhkworld':
 154                 # TODO: figure out if this is something to be fixed in urljoin,
 155                 # _parse_smil_formats or keep it here
 156                 transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
 157             formats.extend(self._extract_smil_formats(
 158                 re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
 159                 transform_source=transform_source, fatal=False))
 160
 161         subtitles = {}
 162         for caption in video_data.get('captions', []):
 163             caption_url = caption.get('url')
 164             if caption_url:
 165                 subtitles.setdefault(caption.get('locale', 'en'), []).append({
 166                     'url': caption_url})
 167
 168         return {
 169             'id': video_id,
 170             'title': title,
 171             'description': video_data.get('description'),
 172             'thumbnail': video_data.get('thumbnailUrl'),
 173             'timestamp': parse_iso8601(video_data.get('dateadd')),
 174             'formats': formats,
 175             'subtitles': subtitles,
 176             '_format_sort_fields': ('tbr', ),  # Incomplete resolution information
 177         }