yt_dlp/extractor/adobetv.py

   1 import functools
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..compat import compat_str
   6 from ..utils import (
   7     float_or_none,
   8     int_or_none,
   9     ISO639Utils,
  10     join_nonempty,
  11     OnDemandPagedList,
  12     parse_duration,
  13     str_or_none,
  14     str_to_int,
  15     unified_strdate,
  16 )
  17
  18
  19 class AdobeTVBaseIE(InfoExtractor):
  20     def _call_api(self, path, video_id, query, note=None):
  21         return self._download_json(
  22             'http://tv.adobe.com/api/v4/' + path,
  23             video_id, note, query=query)['data']
  24
  25     def _parse_subtitles(self, video_data, url_key):
  26         subtitles = {}
  27         for translation in video_data.get('translations', []):
  28             vtt_path = translation.get(url_key)
  29             if not vtt_path:
  30                 continue
  31             lang = translation.get('language_w3c') or ISO639Utils.long2short(translation['language_medium'])
  32             subtitles.setdefault(lang, []).append({
  33                 'ext': 'vtt',
  34                 'url': vtt_path,
  35             })
  36         return subtitles
  37
  38     def _parse_video_data(self, video_data):
  39         video_id = compat_str(video_data['id'])
  40         title = video_data['title']
  41
  42         s3_extracted = False
  43         formats = []
  44         for source in video_data.get('videos', []):
  45             source_url = source.get('url')
  46             if not source_url:
  47                 continue
  48             f = {
  49                 'format_id': source.get('quality_level'),
  50                 'fps': int_or_none(source.get('frame_rate')),
  51                 'height': int_or_none(source.get('height')),
  52                 'tbr': int_or_none(source.get('video_data_rate')),
  53                 'width': int_or_none(source.get('width')),
  54                 'url': source_url,
  55             }
  56             original_filename = source.get('original_filename')
  57             if original_filename:
  58                 if not (f.get('height') and f.get('width')):
  59                     mobj = re.search(r'_(\d+)x(\d+)', original_filename)
  60                     if mobj:
  61                         f.update({
  62                             'height': int(mobj.group(2)),
  63                             'width': int(mobj.group(1)),
  64                         })
  65                 if original_filename.startswith('s3://') and not s3_extracted:
  66                     formats.append({
  67                         'format_id': 'original',
  68                         'quality': 1,
  69                         'url': original_filename.replace('s3://', 'https://s3.amazonaws.com/'),
  70                     })
  71                     s3_extracted = True
  72             formats.append(f)
  73
  74         return {
  75             'id': video_id,
  76             'title': title,
  77             'description': video_data.get('description'),
  78             'thumbnail': video_data.get('thumbnail'),
  79             'upload_date': unified_strdate(video_data.get('start_date')),
  80             'duration': parse_duration(video_data.get('duration')),
  81             'view_count': str_to_int(video_data.get('playcount')),
  82             'formats': formats,
  83             'subtitles': self._parse_subtitles(video_data, 'vtt'),
  84         }
  85
  86
  87 class AdobeTVEmbedIE(AdobeTVBaseIE):
  88     IE_NAME = 'adobetv:embed'
  89     _VALID_URL = r'https?://tv\.adobe\.com/embed/\d+/(?P<id>\d+)'
  90     _TEST = {
  91         'url': 'https://tv.adobe.com/embed/22/4153',
  92         'md5': 'c8c0461bf04d54574fc2b4d07ac6783a',
  93         'info_dict': {
  94             'id': '4153',
  95             'ext': 'flv',
  96             'title': 'Creating Graphics Optimized for BlackBerry',
  97             'description': 'md5:eac6e8dced38bdaae51cd94447927459',
  98             'thumbnail': r're:https?://.*\.jpg$',
  99             'upload_date': '20091109',
 100             'duration': 377,
 101             'view_count': int,
 102         },
 103     }
 104
 105     def _real_extract(self, url):
 106         video_id = self._match_id(url)
 107
 108         video_data = self._call_api(
 109             'episode/' + video_id, video_id, {'disclosure': 'standard'})[0]
 110         return self._parse_video_data(video_data)
 111
 112
 113 class AdobeTVIE(AdobeTVBaseIE):
 114     IE_NAME = 'adobetv'
 115     _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?watch/(?P<show_urlname>[^/]+)/(?P<id>[^/]+)'
 116
 117     _TEST = {
 118         'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
 119         'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
 120         'info_dict': {
 121             'id': '10981',
 122             'ext': 'mp4',
 123             'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
 124             'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
 125             'thumbnail': r're:https?://.*\.jpg$',
 126             'upload_date': '20110914',
 127             'duration': 60,
 128             'view_count': int,
 129         },
 130     }
 131
 132     def _real_extract(self, url):
 133         language, show_urlname, urlname = self._match_valid_url(url).groups()
 134         if not language:
 135             language = 'en'
 136
 137         video_data = self._call_api(
 138             'episode/get', urlname, {
 139                 'disclosure': 'standard',
 140                 'language': language,
 141                 'show_urlname': show_urlname,
 142                 'urlname': urlname,
 143             })[0]
 144         return self._parse_video_data(video_data)
 145
 146
 147 class AdobeTVPlaylistBaseIE(AdobeTVBaseIE):
 148     _PAGE_SIZE = 25
 149
 150     def _fetch_page(self, display_id, query, page):
 151         page += 1
 152         query['page'] = page
 153         for element_data in self._call_api(
 154                 self._RESOURCE, display_id, query, 'Download Page %d' % page):
 155             yield self._process_data(element_data)
 156
 157     def _extract_playlist_entries(self, display_id, query):
 158         return OnDemandPagedList(functools.partial(
 159             self._fetch_page, display_id, query), self._PAGE_SIZE)
 160
 161
 162 class AdobeTVShowIE(AdobeTVPlaylistBaseIE):
 163     IE_NAME = 'adobetv:show'
 164     _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?show/(?P<id>[^/]+)'
 165
 166     _TEST = {
 167         'url': 'http://tv.adobe.com/show/the-complete-picture-with-julieanne-kost',
 168         'info_dict': {
 169             'id': '36',
 170             'title': 'The Complete Picture with Julieanne Kost',
 171             'description': 'md5:fa50867102dcd1aa0ddf2ab039311b27',
 172         },
 173         'playlist_mincount': 136,
 174     }
 175     _RESOURCE = 'episode'
 176     _process_data = AdobeTVBaseIE._parse_video_data
 177
 178     def _real_extract(self, url):
 179         language, show_urlname = self._match_valid_url(url).groups()
 180         if not language:
 181             language = 'en'
 182         query = {
 183             'disclosure': 'standard',
 184             'language': language,
 185             'show_urlname': show_urlname,
 186         }
 187
 188         show_data = self._call_api(
 189             'show/get', show_urlname, query)[0]
 190
 191         return self.playlist_result(
 192             self._extract_playlist_entries(show_urlname, query),
 193             str_or_none(show_data.get('id')),
 194             show_data.get('show_name'),
 195             show_data.get('show_description'))
 196
 197
 198 class AdobeTVChannelIE(AdobeTVPlaylistBaseIE):
 199     IE_NAME = 'adobetv:channel'
 200     _VALID_URL = r'https?://tv\.adobe\.com/(?:(?P<language>fr|de|es|jp)/)?channel/(?P<id>[^/]+)(?:/(?P<category_urlname>[^/]+))?'
 201
 202     _TEST = {
 203         'url': 'http://tv.adobe.com/channel/development',
 204         'info_dict': {
 205             'id': 'development',
 206         },
 207         'playlist_mincount': 96,
 208     }
 209     _RESOURCE = 'show'
 210
 211     def _process_data(self, show_data):
 212         return self.url_result(
 213             show_data['url'], 'AdobeTVShow', str_or_none(show_data.get('id')))
 214
 215     def _real_extract(self, url):
 216         language, channel_urlname, category_urlname = self._match_valid_url(url).groups()
 217         if not language:
 218             language = 'en'
 219         query = {
 220             'channel_urlname': channel_urlname,
 221             'language': language,
 222         }
 223         if category_urlname:
 224             query['category_urlname'] = category_urlname
 225
 226         return self.playlist_result(
 227             self._extract_playlist_entries(channel_urlname, query),
 228             channel_urlname)
 229
 230
 231 class AdobeTVVideoIE(AdobeTVBaseIE):
 232     IE_NAME = 'adobetv:video'
 233     _VALID_URL = r'https?://video\.tv\.adobe\.com/v/(?P<id>\d+)'
 234     _EMBED_REGEX = [r'<iframe[^>]+src=[\'"](?P<url>(?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]']
 235
 236     _TEST = {
 237         # From https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners
 238         'url': 'https://video.tv.adobe.com/v/2456/',
 239         'md5': '43662b577c018ad707a63766462b1e87',
 240         'info_dict': {
 241             'id': '2456',
 242             'ext': 'mp4',
 243             'title': 'New experience with Acrobat DC',
 244             'description': 'New experience with Acrobat DC',
 245             'duration': 248.667,
 246         },
 247     }
 248
 249     def _real_extract(self, url):
 250         video_id = self._match_id(url)
 251         webpage = self._download_webpage(url, video_id)
 252
 253         video_data = self._parse_json(self._search_regex(
 254             r'var\s+bridge\s*=\s*([^;]+);', webpage, 'bridged data'), video_id)
 255         title = video_data['title']
 256
 257         formats = []
 258         sources = video_data.get('sources') or []
 259         for source in sources:
 260             source_src = source.get('src')
 261             if not source_src:
 262                 continue
 263             formats.append({
 264                 'filesize': int_or_none(source.get('kilobytes') or None, invscale=1000),
 265                 'format_id': join_nonempty(source.get('format'), source.get('label')),
 266                 'height': int_or_none(source.get('height') or None),
 267                 'tbr': int_or_none(source.get('bitrate') or None),
 268                 'width': int_or_none(source.get('width') or None),
 269                 'url': source_src,
 270             })
 271
 272         # For both metadata and downloaded files the duration varies among
 273         # formats. I just pick the max one
 274         duration = max(filter(None, [
 275             float_or_none(source.get('duration'), scale=1000)
 276             for source in sources]))
 277
 278         return {
 279             'id': video_id,
 280             'formats': formats,
 281             'title': title,
 282             'description': video_data.get('description'),
 283             'thumbnail': video_data.get('video', {}).get('poster'),
 284             'duration': duration,
 285             'subtitles': self._parse_subtitles(video_data, 'vttPath'),
 286         }