yt_dlp/extractor/nfb.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     int_or_none,
   4     join_nonempty,
   5     merge_dicts,
   6     parse_count,
   7     url_or_none,
   8 )
   9 from ..utils.traversal import traverse_obj
  10
  11
  12 class NFBBaseIE(InfoExtractor):
  13     _VALID_URL_BASE = r'https?://(?:www\.)?(?P<site>nfb|onf)\.ca'
  14     _GEO_COUNTRIES = ['CA']
  15
  16     def _extract_ep_data(self, webpage, video_id, fatal=False):
  17         return self._search_json(
  18             r'episodesData\s*:', webpage, 'episode data', video_id, fatal=fatal) or {}
  19
  20     def _extract_ep_info(self, data, video_id, slug=None):
  21         info = traverse_obj(data, (lambda _, v: video_id in v['embed_url'], {
  22             'description': ('description', {str}),
  23             'thumbnail': ('thumbnail_url', {url_or_none}),
  24             'uploader': ('data_layer', 'episodeMaker', {str}),
  25             'release_year': ('data_layer', 'episodeYear', {int_or_none}),
  26             'episode': ('data_layer', 'episodeTitle', {str}),
  27             'season': ('data_layer', 'seasonTitle', {str}),
  28             'season_number': ('data_layer', 'seasonTitle', {parse_count}),
  29             'series': ('data_layer', 'seriesTitle', {str}),
  30         }), get_all=False)
  31
  32         return {
  33             **info,
  34             'id': video_id,
  35             'title': join_nonempty('series', 'episode', from_dict=info, delim=' - '),
  36             'episode_number': int_or_none(self._search_regex(
  37                 r'[/-]e(?:pisode)?-?(\d+)(?:[/-]|$)', slug or video_id, 'episode number', default=None)),
  38         }
  39
  40
  41 class NFBIE(NFBBaseIE):
  42     IE_NAME = 'nfb'
  43     IE_DESC = 'nfb.ca and onf.ca films and episodes'
  44     _VALID_URL = [
  45         rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>film)/(?P<id>[^/?#&]+)',
  46         rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+/s(?:ea|ai)son\d+/episode\d+)',
  47     ]
  48     _TESTS = [{
  49         'note': 'NFB film',
  50         'url': 'https://www.nfb.ca/film/trafficopter/',
  51         'info_dict': {
  52             'id': 'trafficopter',
  53             'ext': 'mp4',
  54             'title': 'Trafficopter',
  55             'description': 'md5:060228455eb85cf88785c41656776bc0',
  56             'thumbnail': r're:^https?://.*\.jpg$',
  57             'uploader': 'Barrie Howells',
  58             'release_year': 1972,
  59             'duration': 600.0,
  60         },
  61         'params': {'skip_download': 'm3u8'},
  62     }, {
  63         'note': 'ONF film',
  64         'url': 'https://www.onf.ca/film/mal-du-siecle/',
  65         'info_dict': {
  66             'id': 'mal-du-siecle',
  67             'ext': 'mp4',
  68             'title': 'Le mal du siècle',
  69             'description': 'md5:1abf774d77569ebe603419f2d344102b',
  70             'thumbnail': r're:^https?://.*\.jpg$',
  71             'uploader': 'Catherine Lepage',
  72             'release_year': 2019,
  73             'duration': 300.0,
  74         },
  75         'params': {'skip_download': 'm3u8'},
  76     }, {
  77         'note': 'NFB episode with English title',
  78         'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/season1/episode9/',
  79         'info_dict': {
  80             'id': 'true-north-episode9-true-north-finale-making-it',
  81             'ext': 'mp4',
  82             'title': 'True North: Inside the Rise of Toronto Basketball - Finale: Making It',
  83             'description': 'We catch up with each player in the midst of their journey as they reflect on their road ahead.',
  84             'series': 'True North: Inside the Rise of Toronto Basketball',
  85             'release_year': 2018,
  86             'season': 'Season 1',
  87             'season_number': 1,
  88             'episode': 'Finale: Making It',
  89             'episode_number': 9,
  90             'uploader': 'Ryan Sidhoo',
  91             'thumbnail': r're:^https?://.*\.jpg$',
  92         },
  93         'params': {'skip_download': 'm3u8'},
  94     }, {
  95         'note': 'ONF episode with French title',
  96         'url': 'https://www.onf.ca/serie/direction-nord-la-montee-du-basketball-a-toronto/saison1/episode9/',
  97         'info_dict': {
  98             'id': 'direction-nord-episode-9',
  99             'ext': 'mp4',
 100             'title': 'Direction nord – La montée du basketball à Toronto - Finale : Réussir',
 101             'description': 'md5:349a57419b71432b97bf6083d92b029d',
 102             'series': 'Direction nord – La montée du basketball à Toronto',
 103             'release_year': 2018,
 104             'season': 'Saison 1',
 105             'season_number': 1,
 106             'episode': 'Finale : Réussir',
 107             'episode_number': 9,
 108             'uploader': 'Ryan Sidhoo',
 109             'thumbnail': r're:^https?://.*\.jpg$',
 110         },
 111         'params': {'skip_download': 'm3u8'},
 112     }, {
 113         'note': 'NFB episode with French title (needs geo-bypass)',
 114         'url': 'https://www.nfb.ca/series/etoile-du-nord/saison1/episode1/',
 115         'info_dict': {
 116             'id': 'etoile-du-nord-episode-1-lobservation',
 117             'ext': 'mp4',
 118             'title': 'Étoile du Nord - L\'observation',
 119             'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
 120             'series': 'Étoile du Nord',
 121             'release_year': 2023,
 122             'season': 'Saison 1',
 123             'season_number': 1,
 124             'episode': 'L\'observation',
 125             'episode_number': 1,
 126             'uploader': 'Patrick Bossé',
 127             'thumbnail': r're:^https?://.*\.jpg$',
 128         },
 129         'params': {'skip_download': 'm3u8'},
 130     }, {
 131         'note': 'ONF episode with English title (needs geo-bypass)',
 132         'url': 'https://www.onf.ca/serie/north-star/season1/episode1/',
 133         'info_dict': {
 134             'id': 'north-star-episode-1-observation',
 135             'ext': 'mp4',
 136             'title': 'North Star - Observation',
 137             'description': 'md5:c727f370839d8a817392b9e3f23655c7',
 138             'series': 'North Star',
 139             'release_year': 2023,
 140             'season': 'Season 1',
 141             'season_number': 1,
 142             'episode': 'Observation',
 143             'episode_number': 1,
 144             'uploader': 'Patrick Bossé',
 145             'thumbnail': r're:^https?://.*\.jpg$',
 146         },
 147         'params': {'skip_download': 'm3u8'},
 148     }, {
 149         'note': 'NFB episode with /film/ URL and English title (needs geo-bypass)',
 150         'url': 'https://www.nfb.ca/film/north-star-episode-1-observation/',
 151         'info_dict': {
 152             'id': 'north-star-episode-1-observation',
 153             'ext': 'mp4',
 154             'title': 'North Star - Observation',
 155             'description': 'md5:c727f370839d8a817392b9e3f23655c7',
 156             'series': 'North Star',
 157             'release_year': 2023,
 158             'season': 'Season 1',
 159             'season_number': 1,
 160             'episode': 'Observation',
 161             'episode_number': 1,
 162             'uploader': 'Patrick Bossé',
 163             'thumbnail': r're:^https?://.*\.jpg$',
 164         },
 165         'params': {'skip_download': 'm3u8'},
 166     }, {
 167         'note': 'ONF episode with /film/ URL and French title (needs geo-bypass)',
 168         'url': 'https://www.onf.ca/film/etoile-du-nord-episode-1-lobservation/',
 169         'info_dict': {
 170             'id': 'etoile-du-nord-episode-1-lobservation',
 171             'ext': 'mp4',
 172             'title': 'Étoile du Nord - L\'observation',
 173             'description': 'md5:161a4617260dee3de70f509b2c9dd21b',
 174             'series': 'Étoile du Nord',
 175             'release_year': 2023,
 176             'season': 'Saison 1',
 177             'season_number': 1,
 178             'episode': 'L\'observation',
 179             'episode_number': 1,
 180             'uploader': 'Patrick Bossé',
 181             'thumbnail': r're:^https?://.*\.jpg$',
 182         },
 183         'params': {'skip_download': 'm3u8'},
 184     }, {
 185         'note': 'Season 2 episode w/o episode num in id, extract from json ld',
 186         'url': 'https://www.onf.ca/film/liste-des-choses-qui-existent-saison-2-ours',
 187         'info_dict': {
 188             'id': 'liste-des-choses-qui-existent-saison-2-ours',
 189             'ext': 'mp4',
 190             'title': 'La liste des choses qui existent - L\'ours en peluche',
 191             'description': 'md5:d5e8d8fc5f3a7385a9cf0f509b37e28a',
 192             'series': 'La liste des choses qui existent',
 193             'release_year': 2022,
 194             'season': 'Saison 2',
 195             'season_number': 2,
 196             'episode': 'L\'ours en peluche',
 197             'episode_number': 12,
 198             'uploader': 'Francis Papillon',
 199             'thumbnail': r're:^https?://.*\.jpg$',
 200         },
 201         'params': {'skip_download': 'm3u8'},
 202     }, {
 203         'note': 'NFB film /embed/player/ page',
 204         'url': 'https://www.nfb.ca/film/afterlife/embed/player/',
 205         'info_dict': {
 206             'id': 'afterlife',
 207             'ext': 'mp4',
 208             'title': 'Afterlife',
 209             'description': 'md5:84951394f594f1fb1e62d9c43242fdf5',
 210             'release_year': 1978,
 211             'duration': 420.0,
 212             'uploader': 'Ishu Patel',
 213             'thumbnail': r're:^https?://.*\.jpg$',
 214         },
 215         'params': {'skip_download': 'm3u8'},
 216     }]
 217
 218     def _real_extract(self, url):
 219         site, type_, slug = self._match_valid_url(url).group('site', 'type', 'id')
 220         # Need to construct the URL since we match /embed/player/ URLs as well
 221         webpage, urlh = self._download_webpage_handle(f'https://www.{site}.ca/{type_}/{slug}/', slug)
 222         # type_ can change from film to serie(s) after redirect; new slug may have episode number
 223         type_, slug = self._match_valid_url(urlh.url).group('type', 'id')
 224
 225         player_data = self._search_json(
 226             r'window\.PLAYER_OPTIONS\[[^\]]+\]\s*=', webpage, 'player data', slug)
 227         video_id = self._match_id(player_data['overlay']['url'])  # overlay url always has unique slug
 228
 229         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 230             player_data['source'], video_id, 'mp4', m3u8_id='hls')
 231
 232         if dv_source := url_or_none(player_data.get('dvSource')):
 233             fmts, subs = self._extract_m3u8_formats_and_subtitles(
 234                 dv_source, video_id, 'mp4', m3u8_id='dv', preference=-2, fatal=False)
 235             for fmt in fmts:
 236                 fmt['format_note'] = 'described video'
 237             formats.extend(fmts)
 238             self._merge_subtitles(subs, target=subtitles)
 239
 240         info = {
 241             'id': video_id,
 242             'title': self._html_search_regex(
 243                 r'["\']nfb_version_title["\']\s*:\s*["\']([^"\']+)',
 244                 webpage, 'title', default=None),
 245             'description': self._html_search_regex(
 246                 r'<[^>]+\bid=["\']tabSynopsis["\'][^>]*>\s*<p[^>]*>\s*([^<]+)',
 247                 webpage, 'description', default=None),
 248             'thumbnail': url_or_none(player_data.get('poster')),
 249             'uploader': self._html_search_regex(
 250                 r'<[^>]+\bitemprop=["\']director["\'][^>]*>([^<]+)', webpage, 'uploader', default=None),
 251             'release_year': int_or_none(self._html_search_regex(
 252                 r'["\']nfb_version_year["\']\s*:\s*["\']([^"\']+)',
 253                 webpage, 'release_year', default=None)),
 254         } if type_ == 'film' else self._extract_ep_info(self._extract_ep_data(webpage, video_id, slug), video_id)
 255
 256         return merge_dicts({
 257             'formats': formats,
 258             'subtitles': subtitles,
 259         }, info, self._search_json_ld(webpage, video_id, default={}))
 260
 261
 262 class NFBSeriesIE(NFBBaseIE):
 263     IE_NAME = 'nfb:series'
 264     IE_DESC = 'nfb.ca and onf.ca series'
 265     _VALID_URL = rf'{NFBBaseIE._VALID_URL_BASE}/(?P<type>series?)/(?P<id>[^/?#&]+)/?(?:[?#]|$)'
 266     _TESTS = [{
 267         'url': 'https://www.nfb.ca/series/true-north-inside-the-rise-of-toronto-basketball/',
 268         'playlist_mincount': 9,
 269         'info_dict': {
 270             'id': 'true-north-inside-the-rise-of-toronto-basketball',
 271         },
 272     }, {
 273         'url': 'https://www.onf.ca/serie/la-liste-des-choses-qui-existent-serie/',
 274         'playlist_mincount': 26,
 275         'info_dict': {
 276             'id': 'la-liste-des-choses-qui-existent-serie',
 277         },
 278     }]
 279
 280     def _entries(self, episodes):
 281         for episode in traverse_obj(episodes, lambda _, v: NFBIE.suitable(v['embed_url'])):
 282             mobj = NFBIE._match_valid_url(episode['embed_url'])
 283             yield self.url_result(
 284                 mobj[0], NFBIE, **self._extract_ep_info([episode], mobj.group('id')))
 285
 286     def _real_extract(self, url):
 287         site, type_, series_id = self._match_valid_url(url).group('site', 'type', 'id')
 288         season_path = 'saison' if type_ == 'serie' else 'season'
 289         webpage = self._download_webpage(
 290             f'https://www.{site}.ca/{type_}/{series_id}/{season_path}1/episode1', series_id)
 291         episodes = self._extract_ep_data(webpage, series_id, fatal=True)
 292
 293         return self.playlist_result(self._entries(episodes), series_id)