yt_dlp/extractor/callin.py

   1 from .common import InfoExtractor
   2 from ..utils import float_or_none, int_or_none, make_archive_id, traverse_obj
   3
   4
   5 class CallinIE(InfoExtractor):
   6     _VALID_URL = r'https?://(?:www\.)?callin\.com/(episode)/(?P<id>[-a-zA-Z]+)'
   7     _TESTS = [{
   8         'url': 'https://www.callin.com/episode/the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
   9         'info_dict': {
  10             'id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
  11             'title': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
  12             'ext': 'ts',
  13             'display_id': 'the-title-ix-regime-and-the-long-march-through-EBfXYSrsjc',
  14             'thumbnail': 're:https://.+\\.png',
  15             'description': 'First episode',
  16             'uploader': 'Wesley Yang',
  17             'timestamp': 1639404128.65,
  18             'upload_date': '20211213',
  19             'uploader_id': 'wesyang',
  20             'uploader_url': 'http://wesleyyang.substack.com',
  21             'channel': 'Conversations in Year Zero',
  22             'channel_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
  23             'channel_url': 'https://callin.com/show/conversations-in-year-zero-oJNllRFSfx',
  24             'duration': 9951.936,
  25             'view_count': int,
  26             'categories': ['News & Politics', 'History', 'Technology'],
  27             'cast': ['Wesley Yang', 'KC Johnson', 'Gabi Abramovich'],
  28             'series': 'Conversations in Year Zero',
  29             'series_id': '436d1f82ddeb30cd2306ea9156044d8d2cfdc3f1f1552d245117a42173e78553',
  30             'episode': 'The Title IX Regime and the Long March Through and Beyond the Institutions',
  31             'episode_number': 1,
  32             'episode_id': '218b979630a35ead12c6fd096f2996c56c37e4d0dc1f6dc0feada32dcf7b31cd',
  33         },
  34     }, {
  35         'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
  36         'md5': '14ede27ee2c957b7e4db93140fc0745c',
  37         'info_dict': {
  38             'id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
  39             'ext': 'ts',
  40             'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
  41             'description': 'Or, why the government doesn’t like SpaceX',
  42             'channel': 'The Pull Request',
  43             'channel_url': 'https://callin.com/show/the-pull-request-ucnDJmEKAa',
  44             'duration': 3182.472,
  45             'series_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
  46             'uploader_url': 'http://thepullrequest.com',
  47             'upload_date': '20220902',
  48             'episode': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
  49             'display_id': 'fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
  50             'series': 'The Pull Request',
  51             'channel_id': '7e9c23156e4aecfdcaef46bfb2ed7ca268509622ec006c0f0f25d90e34496638',
  52             'view_count': int,
  53             'uploader': 'Antonio García Martínez',
  54             'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/1ade9142625344045dc17cf523469ced1d93610762f4c886d06aa190a2f979e8.png',
  55             'episode_id': 'c3dab47f237bf953d180d3f243477a84302798be0e0b29bc9ade6d60a69f04f5',
  56             'timestamp': 1662100688.005,
  57         },
  58     }, {
  59         'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
  60         'md5': '16f704ddbf82a27e3930533b12062f07',
  61         'info_dict': {
  62             'id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
  63             'ext': 'ts',
  64             'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
  65             'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
  66             'channel': 'The DEBRIEF With Briahna Joy Gray',
  67             'channel_url': 'https://callin.com/show/the-debrief-with-briahna-joy-gray-siiFDzGegm',
  68             'duration': 10043.16,
  69             'series_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
  70             'uploader_url': 'http://patreon.com/badfaithpodcast',
  71             'upload_date': '20220826',
  72             'episode': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
  73             'display_id': 'episode-',
  74             'series': 'The DEBRIEF With Briahna Joy Gray',
  75             'channel_id': '61cea58444465fd26674069703bd8322993bc9e5b4f1a6d0872690554a046ff7',
  76             'view_count': int,
  77             'uploader': 'Briahna Gray',
  78             'thumbnail': 'https://d1z76fhpoqkd01.cloudfront.net/shows/legacy/461ea0d86172cb6aff7d6c80fd49259cf5e64bdf737a4650f8bc24cf392ca218.png',
  79             'episode_id': '8d06f869798f93a7814e380bceabea72d501417e620180416ff6bd510596e83c',
  80             'timestamp': 1661476708.282,
  81         },
  82     }]
  83
  84     def try_get_user_name(self, d):
  85         names = [d.get(n) for n in ('first', 'last')]
  86         if None in names:
  87             return next((n for n in names if n), default=None)
  88         return ' '.join(names)
  89
  90     def _real_extract(self, url):
  91         display_id = self._match_id(url)
  92         webpage = self._download_webpage(url, display_id)
  93
  94         next_data = self._search_nextjs_data(webpage, display_id)
  95         episode = next_data['props']['pageProps']['episode']
  96
  97         video_id = episode['id']
  98         title = episode.get('title') or self._generic_title('', webpage)
  99         url = episode['m3u8']
 100         formats = self._extract_m3u8_formats(url, display_id, ext='ts')
 101
 102         show = traverse_obj(episode, ('show', 'title'))
 103         show_id = traverse_obj(episode, ('show', 'id'))
 104
 105         show_json = None
 106         app_slug = (self._html_search_regex(
 107             '<script\\s+src=["\']/_next/static/([-_a-zA-Z0-9]+)/_',
 108             webpage, 'app slug', fatal=False) or next_data.get('buildId'))
 109         show_slug = traverse_obj(episode, ('show', 'linkObj', 'resourceUrl'))
 110         if app_slug and show_slug and '/' in show_slug:
 111             show_slug = show_slug.rsplit('/', 1)[1]
 112             show_json_url = f'https://www.callin.com/_next/data/{app_slug}/show/{show_slug}.json'
 113             show_json = self._download_json(show_json_url, display_id, fatal=False)
 114
 115         host = (traverse_obj(show_json, ('pageProps', 'show', 'hosts', 0))
 116                 or traverse_obj(episode, ('speakers', 0)))
 117
 118         host_nick = traverse_obj(host, ('linkObj', 'resourceUrl'))
 119         host_nick = host_nick.rsplit('/', 1)[1] if (host_nick and '/' in host_nick) else None
 120
 121         cast = list(filter(None, [
 122             self.try_get_user_name(u) for u in
 123             traverse_obj(episode, (('speakers', 'callerTags'), ...)) or []
 124         ]))
 125
 126         episode_list = traverse_obj(show_json, ('pageProps', 'show', 'episodes')) or []
 127         episode_number = next(
 128             (len(episode_list) - i for i, e in enumerate(episode_list) if e.get('id') == video_id),
 129             None)
 130
 131         return {
 132             'id': video_id,
 133             '_old_archive_ids': [make_archive_id(self, display_id.rsplit('-', 1)[-1])],
 134             'display_id': display_id,
 135             'title': title,
 136             'formats': formats,
 137             'thumbnail': traverse_obj(episode, ('show', 'photo')),
 138             'description': episode.get('description'),
 139             'uploader': self.try_get_user_name(host) if host else None,
 140             'timestamp': episode.get('publishedAt'),
 141             'uploader_id': host_nick,
 142             'uploader_url': traverse_obj(show_json, ('pageProps', 'show', 'url')),
 143             'channel': show,
 144             'channel_id': show_id,
 145             'channel_url': traverse_obj(episode, ('show', 'linkObj', 'resourceUrl')),
 146             'duration': float_or_none(episode.get('runtime')),
 147             'view_count': int_or_none(episode.get('plays')),
 148             'categories': traverse_obj(episode, ('show', 'categorizations', ..., 'name')),
 149             'cast': cast if cast else None,
 150             'series': show,
 151             'series_id': show_id,
 152             'episode': title,
 153             'episode_number': episode_number,
 154             'episode_id': video_id,
 155         }