yt_dlp/extractor/substack.py

   1 import re
   2 import urllib.parse
   3
   4 from .common import InfoExtractor
   5 from ..utils import js_to_json, str_or_none, traverse_obj
   6
   7
   8 class SubstackIE(InfoExtractor):
   9     _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)'
  10     _TESTS = [{
  11         'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r',
  12         'md5': 'f27e4fc6252001d48d479f45e65cdfd5',
  13         'info_dict': {
  14             'id': '47660949',
  15             'ext': 'mp4',
  16             'title': 'I MADE A VLOG',
  17             'description': 'md5:9248af9a759321e1027226f988f54d96',
  18             'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
  19             'uploader': 'Maybe Baby',
  20             'uploader_id': '33628',
  21         },
  22     }, {
  23         'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r',
  24         'md5': '0a63eacec877a1171a62cfa69710fcea',
  25         'info_dict': {
  26             'id': '51045592',
  27             'ext': 'mpga',
  28             'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account",
  29             'description': 'md5:a57f2439319e56e0af92dd0c95d75797',
  30             'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639',
  31             'uploader': 'Maybe Baby',
  32             'uploader_id': '33628',
  33         },
  34     }, {
  35         'url': 'https://andrewzimmern.substack.com/p/mussels-with-black-bean-sauce-recipe',
  36         'md5': 'fd3c07077b02444ff0130715b5f632bb',
  37         'info_dict': {
  38             'id': '47368578',
  39             'ext': 'mp4',
  40             'title': 'Mussels with Black Bean Sauce: Recipe of the Week #7',
  41             'description': 'md5:b96234a2906c7d854d5229818d889515',
  42             'thumbnail': 'md5:e30bfaa9da40e82aa62354263a9dd232',
  43             'uploader': "Andrew Zimmern's Spilled Milk ",
  44             'uploader_id': '577659',
  45         },
  46     }]
  47
  48     @classmethod
  49     def _extract_embed_urls(cls, url, webpage):
  50         if not re.search(r'<script[^>]+src=["\']https://substackcdn.com/[^"\']+\.js', webpage):
  51             return
  52
  53         mobj = re.search(r'{[^}]*\\?["\']subdomain\\?["\']\s*:\s*\\?["\'](?P<subdomain>[^\\"\']+)', webpage)
  54         if mobj:
  55             parsed = urllib.parse.urlparse(url)
  56             yield parsed._replace(netloc=f'{mobj.group("subdomain")}.substack.com').geturl()
  57             raise cls.StopExtraction
  58
  59     def _extract_video_formats(self, video_id, url):
  60         formats, subtitles = [], {}
  61         for video_format in ('hls', 'mp4'):
  62             video_url = urllib.parse.urljoin(url, f'/api/v1/video/upload/{video_id}/src?type={video_format}')
  63
  64             if video_format == 'hls':
  65                 fmts, subs = self._extract_m3u8_formats_and_subtitles(video_url, video_id, 'mp4', fatal=False)
  66                 formats.extend(fmts)
  67                 self._merge_subtitles(subs, target=subtitles)
  68             else:
  69                 formats.append({
  70                     'url': video_url,
  71                     'ext': video_format,
  72                 })
  73
  74         return formats, subtitles
  75
  76     def _real_extract(self, url):
  77         display_id, username = self._match_valid_url(url).group('id', 'username')
  78         webpage = self._download_webpage(url, display_id)
  79
  80         webpage_info = self._parse_json(self._search_json(
  81             r'window\._preloads\s*=\s*JSON\.parse\(', webpage, 'json string',
  82             display_id, transform_source=js_to_json, contains_pattern=r'"{(?s:.+)}"'), display_id)
  83
  84         canonical_url = url
  85         domain = traverse_obj(webpage_info, ('domainInfo', 'customDomain', {str}))
  86         if domain:
  87             canonical_url = urllib.parse.urlparse(url)._replace(netloc=domain).geturl()
  88
  89         post_type = webpage_info['post']['type']
  90         formats, subtitles = [], {}
  91         if post_type == 'podcast':
  92             formats, subtitles = [{'url': webpage_info['post']['podcast_url']}], {}
  93         elif post_type == 'video':
  94             formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], canonical_url)
  95         else:
  96             self.raise_no_formats(f'Page type "{post_type}" is not supported')
  97
  98         return {
  99             'id': str(webpage_info['post']['id']),
 100             'formats': formats,
 101             'subtitles': subtitles,
 102             'title': traverse_obj(webpage_info, ('post', 'title')),
 103             'description': traverse_obj(webpage_info, ('post', 'description')),
 104             'thumbnail': traverse_obj(webpage_info, ('post', 'cover_image')),
 105             'uploader': traverse_obj(webpage_info, ('pub', 'name')),
 106             'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))),
 107             'webpage_url': canonical_url,
 108         }