yt_dlp/extractor/weibo.py

   1 import itertools
   2 import json
   3 import random
   4 import urllib.parse
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     int_or_none,
   9     make_archive_id,
  10     mimetype2ext,
  11     parse_resolution,
  12     str_or_none,
  13     strip_jsonp,
  14     traverse_obj,
  15     url_or_none,
  16     urlencode_postdata,
  17     urljoin,
  18 )
  19
  20
  21 class WeiboBaseIE(InfoExtractor):
  22     def _update_visitor_cookies(self, visitor_url, video_id):
  23         headers = {'Referer': visitor_url}
  24         chrome_ver = self._search_regex(
  25             r'Chrome/(\d+)', self.get_param('http_headers')['User-Agent'], 'user agent version', default='90')
  26         visitor_data = self._download_json(
  27             'https://passport.weibo.com/visitor/genvisitor', video_id,
  28             note='Generating first-visit guest request',
  29             headers=headers, transform_source=strip_jsonp,
  30             data=urlencode_postdata({
  31                 'cb': 'gen_callback',
  32                 'fp': json.dumps({
  33                     'os': '1',
  34                     'browser': f'Chrome{chrome_ver},0,0,0',
  35                     'fonts': 'undefined',
  36                     'screenInfo': '1920*1080*24',
  37                     'plugins': '',
  38                 }, separators=(',', ':'))}))['data']
  39
  40         self._download_webpage(
  41             'https://passport.weibo.com/visitor/visitor', video_id,
  42             note='Running first-visit callback to get guest cookies',
  43             headers=headers, query={
  44                 'a': 'incarnate',
  45                 't': visitor_data['tid'],
  46                 'w': 3 if visitor_data.get('new_tid') else 2,
  47                 'c': f'{visitor_data.get("confidence", 100):03d}',
  48                 'gc': '',
  49                 'cb': 'cross_domain',
  50                 'from': 'weibo',
  51                 '_rand': random.random(),
  52             })
  53
  54     def _weibo_download_json(self, url, video_id, *args, fatal=True, note='Downloading JSON metadata', **kwargs):
  55         # XXX: Always fatal; _download_webpage_handle only returns False (not a tuple) on error
  56         webpage, urlh = self._download_webpage_handle(url, video_id, *args, fatal=fatal, note=note, **kwargs)
  57         if urllib.parse.urlparse(urlh.url).netloc == 'passport.weibo.com':
  58             self._update_visitor_cookies(urlh.url, video_id)
  59             webpage = self._download_webpage(url, video_id, *args, fatal=fatal, note=note, **kwargs)
  60         return self._parse_json(webpage, video_id, fatal=fatal)
  61
  62     def _extract_formats(self, video_info):
  63         media_info = traverse_obj(video_info, ('page_info', 'media_info'))
  64         formats = traverse_obj(media_info, (
  65             'playback_list', lambda _, v: url_or_none(v['play_info']['url']), 'play_info', {
  66                 'url': 'url',
  67                 'format': ('quality_desc', {str}),
  68                 'format_id': ('label', {str}),
  69                 'ext': ('mime', {mimetype2ext}),
  70                 'tbr': ('bitrate', {int_or_none}, {lambda x: x or None}),
  71                 'vcodec': ('video_codecs', {str}),
  72                 'fps': ('fps', {int_or_none}),
  73                 'width': ('width', {int_or_none}),
  74                 'height': ('height', {int_or_none}),
  75                 'filesize': ('size', {int_or_none}),
  76                 'acodec': ('audio_codecs', {str}),
  77                 'asr': ('audio_sample_rate', {int_or_none}),
  78                 'audio_channels': ('audio_channels', {int_or_none}),
  79             }))
  80         if not formats:  # fallback, should be barely used
  81             for url in set(traverse_obj(media_info, (..., {url_or_none}))):
  82                 if 'label=' in url:  # filter out non-video urls
  83                     format_id, resolution = self._search_regex(
  84                         r'label=(\w+)&template=(\d+x\d+)', url, 'format info',
  85                         group=(1, 2), default=(None, None))
  86                     formats.append({
  87                         'url': url,
  88                         'format_id': format_id,
  89                         **parse_resolution(resolution),
  90                         **traverse_obj(media_info, (
  91                             'video_details', lambda _, v: v['label'].startswith(format_id), {
  92                                 'size': ('size', {int_or_none}),
  93                                 'tbr': ('bitrate', {int_or_none}),
  94                             },
  95                         ), get_all=False),
  96                     })
  97         return formats
  98
  99     def _parse_video_info(self, video_info, video_id=None):
 100         return {
 101             'id': video_id,
 102             'extractor_key': WeiboIE.ie_key(),
 103             'extractor': WeiboIE.IE_NAME,
 104             'formats': self._extract_formats(video_info),
 105             'http_headers': {'Referer': 'https://weibo.com/'},
 106             '_old_archive_ids': [make_archive_id('WeiboMobile', video_id)],
 107             **traverse_obj(video_info, {
 108                 'id': (('id', 'id_str', 'mid'), {str_or_none}),
 109                 'display_id': ('mblogid', {str_or_none}),
 110                 'title': ('page_info', 'media_info', ('video_title', 'kol_title', 'name'), {str}, {lambda x: x or None}),
 111                 'description': ('text_raw', {str}),
 112                 'duration': ('page_info', 'media_info', 'duration', {int_or_none}),
 113                 'timestamp': ('page_info', 'media_info', 'video_publish_time', {int_or_none}),
 114                 'thumbnail': ('page_info', 'page_pic', {url_or_none}),
 115                 'uploader': ('user', 'screen_name', {str}),
 116                 'uploader_id': ('user', ('id', 'id_str'), {str_or_none}),
 117                 'uploader_url': ('user', 'profile_url', {lambda x: urljoin('https://weibo.com/', x)}),
 118                 'view_count': ('page_info', 'media_info', 'online_users_number', {int_or_none}),
 119                 'like_count': ('attitudes_count', {int_or_none}),
 120                 'repost_count': ('reposts_count', {int_or_none}),
 121             }, get_all=False),
 122             'tags': traverse_obj(video_info, ('topic_struct', ..., 'topic_title', {str})) or None,
 123         }
 124
 125
 126 class WeiboIE(WeiboBaseIE):
 127     _VALID_URL = r'https?://(?:m\.weibo\.cn/status|(?:www\.)?weibo\.com/\d+)/(?P<id>[a-zA-Z0-9]+)'
 128     _TESTS = [{
 129         'url': 'https://weibo.com/7827771738/N4xlMvjhI',
 130         'info_dict': {
 131             'id': '4910815147462302',
 132             'ext': 'mp4',
 133             'display_id': 'N4xlMvjhI',
 134             'title': '【睡前消息暑假版第一期：拉泰国一把  对中国有好处】',
 135             'description': 'md5:e2637a7673980d68694ea7c43cf12a5f',
 136             'duration': 918,
 137             'timestamp': 1686312819,
 138             'upload_date': '20230609',
 139             'thumbnail': r're:https://.*\.jpg',
 140             'uploader': '睡前视频基地',
 141             'uploader_id': '7827771738',
 142             'uploader_url': 'https://weibo.com/u/7827771738',
 143             'view_count': int,
 144             'like_count': int,
 145             'repost_count': int,
 146             'tags': ['泰国大选远进党获胜', '睡前消息', '暑期版'],
 147         },
 148     }, {
 149         'url': 'https://m.weibo.cn/status/4189191225395228',
 150         'info_dict': {
 151             'id': '4189191225395228',
 152             'ext': 'mp4',
 153             'display_id': 'FBqgOmDxO',
 154             'title': '柴犬柴犬的秒拍视频',
 155             'description': 'md5:80f461ab5cdae6bbdb70efbf5a1db24f',
 156             'duration': 53,
 157             'timestamp': 1514264429,
 158             'upload_date': '20171226',
 159             'thumbnail': r're:https://.*\.jpg',
 160             'uploader': '柴犬柴犬',
 161             'uploader_id': '5926682210',
 162             'uploader_url': 'https://weibo.com/u/5926682210',
 163             'view_count': int,
 164             'like_count': int,
 165             'repost_count': int,
 166         },
 167     }, {
 168         'url': 'https://weibo.com/0/4224132150961381',
 169         'note': 'no playback_list example',
 170         'only_matching': True,
 171     }]
 172
 173     def _real_extract(self, url):
 174         video_id = self._match_id(url)
 175
 176         return self._parse_video_info(self._weibo_download_json(
 177             f'https://weibo.com/ajax/statuses/show?id={video_id}', video_id))
 178
 179
 180 class WeiboVideoIE(WeiboBaseIE):
 181     _VALID_URL = r'https?://(?:www\.)?weibo\.com/tv/show/(?P<id>\d+:\d+)'
 182     _TESTS = [{
 183         'url': 'https://weibo.com/tv/show/1034:4797699866951785?from=old_pc_videoshow',
 184         'info_dict': {
 185             'id': '4797700463137878',
 186             'ext': 'mp4',
 187             'display_id': 'LEZDodaiW',
 188             'title': '呃，稍微了解了一下靡烟miya，感觉这东西也太二了',
 189             'description': '呃，稍微了解了一下靡烟miya，感觉这东西也太二了 http://t.cn/A6aerGsM \u200b\u200b\u200b',
 190             'duration': 76,
 191             'timestamp': 1659344278,
 192             'upload_date': '20220801',
 193             'thumbnail': r're:https://.*\.jpg',
 194             'uploader': '君子爱财陈平安',
 195             'uploader_id': '3905382233',
 196             'uploader_url': 'https://weibo.com/u/3905382233',
 197             'view_count': int,
 198             'like_count': int,
 199             'repost_count': int,
 200         },
 201     }]
 202
 203     def _real_extract(self, url):
 204         video_id = self._match_id(url)
 205
 206         post_data = f'data={{"Component_Play_Playinfo":{{"oid":"{video_id}"}}}}'.encode()
 207         video_info = self._weibo_download_json(
 208             f'https://weibo.com/tv/api/component?page=%2Ftv%2Fshow%2F{video_id.replace(":", "%3A")}',
 209             video_id, headers={'Referer': url}, data=post_data)['data']['Component_Play_Playinfo']
 210         return self.url_result(f'https://weibo.com/0/{video_info["mid"]}', WeiboIE)
 211
 212
 213 class WeiboUserIE(WeiboBaseIE):
 214     _VALID_URL = r'https?://(?:www\.)?weibo\.com/u/(?P<id>\d+)'
 215     _TESTS = [{
 216         'url': 'https://weibo.com/u/2066652961?tabtype=video',
 217         'info_dict': {
 218             'id': '2066652961',
 219             'title': '萧影殿下的视频',
 220             'description': '萧影殿下的全部视频',
 221             'uploader': '萧影殿下',
 222         },
 223         'playlist_mincount': 195,
 224     }]
 225
 226     def _fetch_page(self, uid, cursor=0, page=1):
 227         return self._weibo_download_json(
 228             'https://weibo.com/ajax/profile/getWaterFallContent',
 229             uid, note=f'Downloading videos page {page}',
 230             query={'uid': uid, 'cursor': cursor})['data']
 231
 232     def _entries(self, uid, first_page):
 233         cursor = 0
 234         for page in itertools.count(1):
 235             response = first_page if page == 1 else self._fetch_page(uid, cursor, page)
 236             for video_info in traverse_obj(response, ('list', ..., {dict})):
 237                 yield self._parse_video_info(video_info)
 238             cursor = response.get('next_cursor')
 239             if (int_or_none(cursor) or -1) < 0:
 240                 break
 241
 242     def _real_extract(self, url):
 243         uid = self._match_id(url)
 244         first_page = self._fetch_page(uid)
 245         uploader = traverse_obj(first_page, ('list', ..., 'user', 'screen_name', {str}), get_all=False)
 246         metainfo = {
 247             'title': f'{uploader}的视频',
 248             'description': f'{uploader}的全部视频',
 249             'uploader': uploader,
 250         } if uploader else {}
 251
 252         return self.playlist_result(self._entries(uid, first_page), uid, **metainfo)