yt_dlp/extractor/cspan.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from .senategov import SenateISVPIE
   5 from .ustream import UstreamIE
   6 from ..compat import compat_HTMLParseError
   7 from ..utils import (
   8     ExtractorError,
   9     determine_ext,
  10     extract_attributes,
  11     find_xpath_attr,
  12     get_element_by_attribute,
  13     get_element_by_class,
  14     int_or_none,
  15     join_nonempty,
  16     js_to_json,
  17     merge_dicts,
  18     parse_iso8601,
  19     parse_qs,
  20     smuggle_url,
  21     str_to_int,
  22     unescapeHTML,
  23 )
  24
  25
  26 class CSpanIE(InfoExtractor):
  27     _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
  28     IE_DESC = 'C-SPAN'
  29     _TESTS = [{
  30         'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
  31         'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
  32         'info_dict': {
  33             'id': '315139',
  34             'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
  35         },
  36         'playlist_mincount': 2,
  37         'skip': 'Regularly fails on travis, for unknown reasons',
  38     }, {
  39         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
  40         # md5 is unstable
  41         'info_dict': {
  42             'id': 'c4486943',
  43             'ext': 'mp4',
  44             'title': 'CSPAN - International Health Care Models',
  45             'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
  46         },
  47     }, {
  48         'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
  49         'info_dict': {
  50             'id': '342759',
  51             'title': 'General Motors Ignition Switch Recall',
  52         },
  53         'playlist_mincount': 6,
  54     }, {
  55         # Video from senate.gov
  56         'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
  57         'info_dict': {
  58             'id': 'judiciary031715',
  59             'ext': 'mp4',
  60             'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
  61         },
  62         'params': {
  63             'skip_download': True,  # m3u8 downloads
  64         },
  65     }, {
  66         # Ustream embedded video
  67         'url': 'https://www.c-span.org/video/?114917-1/armed-services',
  68         'info_dict': {
  69             'id': '58428542',
  70             'ext': 'flv',
  71             'title': 'USHR07 Armed Services Committee',
  72             'description': 'hsas00-2118-20150204-1000et-07\n\n\nUSHR07 Armed Services Committee',
  73             'timestamp': 1423060374,
  74             'upload_date': '20150204',
  75             'uploader': 'HouseCommittee',
  76             'uploader_id': '12987475',
  77         },
  78     }, {
  79         # Audio Only
  80         'url': 'https://www.c-span.org/video/?437336-1/judiciary-antitrust-competition-policy-consumer-rights',
  81         'only_matching': True,
  82     }]
  83     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
  84
  85     def _real_extract(self, url):
  86         video_id = self._match_id(url)
  87         video_type = None
  88         webpage = self._download_webpage(url, video_id)
  89
  90         ustream_url = UstreamIE._extract_url(webpage)
  91         if ustream_url:
  92             return self.url_result(ustream_url, UstreamIE.ie_key())
  93
  94         if '&vod' not in url:
  95             bc = self._search_regex(
  96                 r"(<[^>]+id='brightcove-player-embed'[^>]+>)",
  97                 webpage, 'brightcove embed', default=None)
  98             if bc:
  99                 bc_attr = extract_attributes(bc)
 100                 bc_url = self.BRIGHTCOVE_URL_TEMPLATE % (
 101                     bc_attr.get('data-bcaccountid', '3162030207001'),
 102                     bc_attr.get('data-noprebcplayerid', 'SyGGpuJy3g'),
 103                     bc_attr.get('data-newbcplayerid', 'default'),
 104                     bc_attr['data-bcid'])
 105                 return self.url_result(smuggle_url(bc_url, {'source_url': url}))
 106
 107         def add_referer(formats):
 108             for f in formats:
 109                 f.setdefault('http_headers', {})['Referer'] = url
 110
 111         # As of 01.12.2020 this path looks to cover all cases making the rest
 112         # of the code unnecessary
 113         jwsetup = self._parse_json(
 114             self._search_regex(
 115                 r'(?s)jwsetup\s*=\s*({.+?})\s*;', webpage, 'jwsetup',
 116                 default='{}'),
 117             video_id, transform_source=js_to_json, fatal=False)
 118         if jwsetup:
 119             info = self._parse_jwplayer_data(
 120                 jwsetup, video_id, require_title=False, m3u8_id='hls',
 121                 base_url=url)
 122             add_referer(info['formats'])
 123             for subtitles in info['subtitles'].values():
 124                 for subtitle in subtitles:
 125                     ext = determine_ext(subtitle['url'])
 126                     if ext == 'php':
 127                         ext = 'vtt'
 128                     subtitle['ext'] = ext
 129             ld_info = self._search_json_ld(webpage, video_id, default={})
 130             try:
 131                 title = get_element_by_class('video-page-title', webpage)
 132             except compat_HTMLParseError:
 133                 title = None
 134             if title is None:
 135                 title = self._og_search_title(webpage)
 136             description = get_element_by_attribute('itemprop', 'description', webpage) or \
 137                 self._html_search_meta(['og:description', 'description'], webpage)
 138             return merge_dicts(info, ld_info, {
 139                 'title': title,
 140                 'thumbnail': get_element_by_attribute('itemprop', 'thumbnailUrl', webpage),
 141                 'description': description,
 142                 'timestamp': parse_iso8601(get_element_by_attribute('itemprop', 'uploadDate', webpage)),
 143                 'location': get_element_by_attribute('itemprop', 'contentLocation', webpage),
 144                 'duration': int_or_none(self._search_regex(
 145                     r'jwsetup\.seclength\s*=\s*(\d+);',
 146                     webpage, 'duration', fatal=False)),
 147                 'view_count': str_to_int(self._search_regex(
 148                     r"<span[^>]+class='views'[^>]*>([\d,]+)\s+Views</span>",
 149                     webpage, 'views', fatal=False)),
 150             })
 151
 152         # Obsolete
 153         # We first look for clipid, because clipprog always appears before
 154         patterns = [rf'id=\'clip({t})\'\s*value=\'([0-9]+)\'' for t in ('id', 'prog')]
 155         results = list(filter(None, (re.search(p, webpage) for p in patterns)))
 156         if results:
 157             matches = results[0]
 158             video_type, video_id = matches.groups()
 159             video_type = 'clip' if video_type == 'id' else 'program'
 160         else:
 161             m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
 162             if m:
 163                 video_id = m.group('id')
 164                 video_type = 'program' if m.group('type') == 'prog' else 'clip'
 165             else:
 166                 senate_isvp_url = SenateISVPIE._extract_url(webpage)
 167                 if senate_isvp_url:
 168                     title = self._og_search_title(webpage)
 169                     surl = smuggle_url(senate_isvp_url, {'force_title': title})
 170                     return self.url_result(surl, 'SenateISVP', video_id, title)
 171                 video_id = self._search_regex(
 172                     r'jwsetup\.clipprog\s*=\s*(\d+);',
 173                     webpage, 'jwsetup program id', default=None)
 174                 if video_id:
 175                     video_type = 'program'
 176         if video_type is None or video_id is None:
 177             error_message = get_element_by_class('VLplayer-error-message', webpage)
 178             if error_message:
 179                 raise ExtractorError(error_message)
 180             raise ExtractorError('unable to find video id and type')
 181
 182         def get_text_attr(d, attr):
 183             return d.get(attr, {}).get('#text')
 184
 185         data = self._download_json(
 186             f'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5={video_type}&id={video_id}',
 187             video_id)['video']
 188         if data['@status'] != 'Success':
 189             raise ExtractorError('{} said: {}'.format(self.IE_NAME, get_text_attr(data, 'error')), expected=True)
 190
 191         doc = self._download_xml(
 192             f'http://www.c-span.org/common/services/flashXml.php?{video_type}id={video_id}',
 193             video_id)
 194
 195         description = self._html_search_meta('description', webpage)
 196
 197         title = find_xpath_attr(doc, './/string', 'name', 'title').text
 198         thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
 199
 200         files = data['files']
 201         capfile = get_text_attr(data, 'capfile')
 202
 203         entries = []
 204         for partnum, f in enumerate(files):
 205             formats = []
 206             for quality in f.get('qualities', []):
 207                 formats.append({
 208                     'format_id': '{}-{}p'.format(get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
 209                     'url': unescapeHTML(get_text_attr(quality, 'file')),
 210                     'height': int_or_none(get_text_attr(quality, 'height')),
 211                     'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
 212                 })
 213             if not formats:
 214                 path = unescapeHTML(get_text_attr(f, 'path'))
 215                 if not path:
 216                     continue
 217                 formats = self._extract_m3u8_formats(
 218                     path, video_id, 'mp4', entry_protocol='m3u8_native',
 219                     m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path}]
 220             add_referer(formats)
 221             entries.append({
 222                 'id': f'{video_id}_{partnum + 1}',
 223                 'title': (
 224                     title if len(files) == 1 else
 225                     f'{title} part {partnum + 1}'),
 226                 'formats': formats,
 227                 'description': description,
 228                 'thumbnail': thumbnail,
 229                 'duration': int_or_none(get_text_attr(f, 'length')),
 230                 'subtitles': {
 231                     'en': [{
 232                         'url': capfile,
 233                         'ext': determine_ext(capfile, 'dfxp'),
 234                     }],
 235                 } if capfile else None,
 236             })
 237
 238         if len(entries) == 1:
 239             entry = dict(entries[0])
 240             entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
 241             return entry
 242         else:
 243             return {
 244                 '_type': 'playlist',
 245                 'entries': entries,
 246                 'title': title,
 247                 'id': 'c' + video_id if video_type == 'clip' else video_id,
 248             }
 249
 250
 251 class CSpanCongressIE(InfoExtractor):
 252     _VALID_URL = r'https?://(?:www\.)?c-span\.org/congress/'
 253     _TESTS = [{
 254         'url': 'https://www.c-span.org/congress/?chamber=house&date=2017-12-13&t=1513208380',
 255         'info_dict': {
 256             'id': 'house_2017-12-13',
 257             'title': 'Congressional Chronicle - Members of Congress, Hearings and More',
 258             'description': 'md5:54c264b7a8f219937987610243305a84',
 259             'thumbnail': r're:https://ximage.c-spanvideo.org/.+',
 260             'ext': 'mp4',
 261         },
 262     }]
 263
 264     def _real_extract(self, url):
 265         query = parse_qs(url)
 266         video_date = query.get('date', [None])[0]
 267         video_id = join_nonempty(query.get('chamber', ['senate'])[0], video_date, delim='_')
 268         webpage = self._download_webpage(url, video_id)
 269         if not video_date:
 270             jwp_date = re.search(r'jwsetup.clipprogdate = \'(?P<date>\d{4}-\d{2}-\d{2})\';', webpage)
 271             if jwp_date:
 272                 video_id = f'{video_id}_{jwp_date.group("date")}'
 273         jwplayer_data = self._parse_json(
 274             self._search_regex(r'jwsetup\s*=\s*({(?:.|\n)[^;]+});', webpage, 'player config'),
 275             video_id, transform_source=js_to_json)
 276
 277         title = self._generic_title('', webpage)
 278         description = (self._og_search_description(webpage, default=None)
 279                        or self._html_search_meta('description', webpage, 'description', default=None))
 280
 281         return {
 282             **self._parse_jwplayer_data(jwplayer_data, video_id, False),
 283             'title': re.sub(r'\s+', ' ', title.split('|')[0]).strip(),
 284             'description': description,
 285             'http_headers': {'Referer': 'https://www.c-span.org/'},
 286         }