yt_dlp/extractor/europa.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     int_or_none,
   4     orderedSet,
   5     parse_duration,
   6     parse_iso8601,
   7     parse_qs,
   8     qualities,
   9     traverse_obj,
  10     unified_strdate,
  11     xpath_text,
  12 )
  13
  14
  15 class EuropaIE(InfoExtractor):
  16     _WORKING = False
  17     _VALID_URL = r'https?://ec\.europa\.eu/avservices/(?:video/player|audio/audioDetails)\.cfm\?.*?\bref=(?P<id>[A-Za-z0-9-]+)'
  18     _TESTS = [{
  19         'url': 'http://ec.europa.eu/avservices/video/player.cfm?ref=I107758',
  20         'md5': '574f080699ddd1e19a675b0ddf010371',
  21         'info_dict': {
  22             'id': 'I107758',
  23             'ext': 'mp4',
  24             'title': 'TRADE - Wikileaks on TTIP',
  25             'description': 'NEW  LIVE EC Midday press briefing of 11/08/2015',
  26             'thumbnail': r're:^https?://.*\.jpg$',
  27             'upload_date': '20150811',
  28             'duration': 34,
  29             'view_count': int,
  30             'formats': 'mincount:3',
  31         },
  32     }, {
  33         'url': 'http://ec.europa.eu/avservices/video/player.cfm?sitelang=en&ref=I107786',
  34         'only_matching': True,
  35     }, {
  36         'url': 'http://ec.europa.eu/avservices/audio/audioDetails.cfm?ref=I-109295&sitelang=en',
  37         'only_matching': True,
  38     }]
  39
  40     def _real_extract(self, url):
  41         video_id = self._match_id(url)
  42
  43         playlist = self._download_xml(
  44             f'http://ec.europa.eu/avservices/video/player/playlist.cfm?ID={video_id}', video_id)
  45
  46         def get_item(type_, preference):
  47             items = {}
  48             for item in playlist.findall(f'./info/{type_}/item'):
  49                 lang, label = xpath_text(item, 'lg', default=None), xpath_text(item, 'label', default=None)
  50                 if lang and label:
  51                     items[lang] = label.strip()
  52             for p in preference:
  53                 if items.get(p):
  54                     return items[p]
  55
  56         query = parse_qs(url)
  57         preferred_lang = query.get('sitelang', ('en', ))[0]
  58
  59         preferred_langs = orderedSet((preferred_lang, 'en', 'int'))
  60
  61         title = get_item('title', preferred_langs) or video_id
  62         description = get_item('description', preferred_langs)
  63         thumbnail = xpath_text(playlist, './info/thumburl', 'thumbnail')
  64         upload_date = unified_strdate(xpath_text(playlist, './info/date', 'upload date'))
  65         duration = parse_duration(xpath_text(playlist, './info/duration', 'duration'))
  66         view_count = int_or_none(xpath_text(playlist, './info/views', 'views'))
  67
  68         language_preference = qualities(preferred_langs[::-1])
  69
  70         formats = []
  71         for file_ in playlist.findall('./files/file'):
  72             video_url = xpath_text(file_, './url')
  73             if not video_url:
  74                 continue
  75             lang = xpath_text(file_, './lg')
  76             formats.append({
  77                 'url': video_url,
  78                 'format_id': lang,
  79                 'format_note': xpath_text(file_, './lglabel'),
  80                 'language_preference': language_preference(lang),
  81             })
  82
  83         return {
  84             'id': video_id,
  85             'title': title,
  86             'description': description,
  87             'thumbnail': thumbnail,
  88             'upload_date': upload_date,
  89             'duration': duration,
  90             'view_count': view_count,
  91             'formats': formats,
  92         }
  93
  94
  95 class EuroParlWebstreamIE(InfoExtractor):
  96     _VALID_URL = r'''(?x)
  97         https?://multimedia\.europarl\.europa\.eu/
  98         (?:\w+/)?webstreaming/(?:[\w-]+_)?(?P<id>[\w-]+)
  99     '''
 100     _TESTS = [{
 101         'url': 'https://multimedia.europarl.europa.eu/pl/webstreaming/plenary-session_20220914-0900-PLENARY',
 102         'info_dict': {
 103             'id': '62388b15-d85b-4add-99aa-ba12ccf64f0d',
 104             'display_id': '20220914-0900-PLENARY',
 105             'ext': 'mp4',
 106             'title': 'Plenary session',
 107             'release_timestamp': 1663139069,
 108             'release_date': '20220914',
 109         },
 110         'params': {
 111             'skip_download': True,
 112         },
 113     }, {
 114         # live webstream
 115         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/euroscola_20221115-1000-SPECIAL-EUROSCOLA',
 116         'info_dict': {
 117             'ext': 'mp4',
 118             'id': '510eda7f-ba72-161b-7ee7-0e836cd2e715',
 119             'release_timestamp': 1668502800,
 120             'title': 'Euroscola 2022-11-15 19:21',
 121             'release_date': '20221115',
 122             'live_status': 'is_live',
 123         },
 124         'skip': 'not live anymore',
 125     }, {
 126         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-culture-and-education_20230301-1130-COMMITTEE-CULT',
 127         'info_dict': {
 128             'id': '7355662c-8eac-445e-4bb9-08db14b0ddd7',
 129             'display_id': '20230301-1130-COMMITTEE-CULT',
 130             'ext': 'mp4',
 131             'release_date': '20230301',
 132             'title': 'Committee on Culture and Education',
 133             'release_timestamp': 1677666641,
 134         },
 135     }, {
 136         # live stream
 137         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/committee-on-environment-public-health-and-food-safety_20230524-0900-COMMITTEE-ENVI',
 138         'info_dict': {
 139             'id': 'e4255f56-10aa-4b3c-6530-08db56d5b0d9',
 140             'ext': 'mp4',
 141             'release_date': '20230524',
 142             'title': r're:Committee on Environment, Public Health and Food Safety \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}',
 143             'release_timestamp': 1684911541,
 144             'live_status': 'is_live',
 145         },
 146         'skip': 'Not live anymore',
 147     }, {
 148         'url': 'https://multimedia.europarl.europa.eu/en/webstreaming/20240320-1345-SPECIAL-PRESSER',
 149         'info_dict': {
 150             'id': 'c1f11567-5b52-470a-f3e1-08dc3c216ace',
 151             'display_id': '20240320-1345-SPECIAL-PRESSER',
 152             'ext': 'mp4',
 153             'release_date': '20240320',
 154             'title': 'md5:7c6c814cac55dea5e2d87bf8d3db2234',
 155             'release_timestamp': 1710939767,
 156         },
 157     }, {
 158         'url': 'https://multimedia.europarl.europa.eu/webstreaming/briefing-for-media-on-2024-european-elections_20240429-1000-SPECIAL-OTHER',
 159         'only_matching': True,
 160     }]
 161
 162     def _real_extract(self, url):
 163         display_id = self._match_id(url)
 164         webpage = self._download_webpage(url, display_id)
 165
 166         webpage_nextjs = self._search_nextjs_data(webpage, display_id)['props']['pageProps']
 167
 168         json_info = self._download_json(
 169             'https://acs-api.europarl.connectedviews.eu/api/FullMeeting', display_id,
 170             query={
 171                 'api-version': 1.0,
 172                 'tenantId': 'bae646ca-1fc8-4363-80ba-2c04f06b4968',
 173                 'externalReference': display_id,
 174             })
 175
 176         formats, subtitles = [], {}
 177         for hls_url in traverse_obj(json_info, ((('meetingVideo'), ('meetingVideos', ...)), 'hlsUrl')):
 178             fmt, subs = self._extract_m3u8_formats_and_subtitles(hls_url, display_id)
 179             formats.extend(fmt)
 180             self._merge_subtitles(subs, target=subtitles)
 181
 182         return {
 183             'id': json_info['id'],
 184             'display_id': display_id,
 185             'title': traverse_obj(webpage_nextjs, (('mediaItem', 'title'), ('title', )), get_all=False),
 186             'formats': formats,
 187             'subtitles': subtitles,
 188             'release_timestamp': parse_iso8601(json_info.get('startDateTime')),
 189             'is_live': traverse_obj(webpage_nextjs, ('mediaItem', 'mediaSubType')) == 'Live',
 190         }