yt_dlp/extractor/gettr.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     ExtractorError,
   4     bool_or_none,
   5     dict_get,
   6     float_or_none,
   7     int_or_none,
   8     str_or_none,
   9     traverse_obj,
  10     try_get,
  11     url_or_none,
  12     urljoin,
  13 )
  14
  15
  16 class GettrBaseIE(InfoExtractor):
  17     _BASE_REGEX = r'https?://(www\.)?gettr\.com/'
  18     _MEDIA_BASE_URL = 'https://media.gettr.com/'
  19
  20     def _call_api(self, path, video_id, *args, **kwargs):
  21         return self._download_json(urljoin('https://api.gettr.com/u/', path), video_id, *args, **kwargs)['result']
  22
  23
  24 class GettrIE(GettrBaseIE):
  25     _VALID_URL = GettrBaseIE._BASE_REGEX + r'post/(?P<id>[a-z0-9]+)'
  26
  27     _TESTS = [{
  28         'url': 'https://www.gettr.com/post/pcf6uv838f',
  29         'info_dict': {
  30             'id': 'pcf6uv838f',
  31             'title': 'md5:9086a646bbd06c41c4fe8e52b3c93454',
  32             'description': 'md5:be0577f1e4caadc06de4a002da2bf287',
  33             'ext': 'mp4',
  34             'uploader': 'EpochTV',
  35             'uploader_id': 'epochtv',
  36             'upload_date': '20210927',
  37             'thumbnail': r're:^https?://.+/out\.jpg',
  38             'timestamp': 1632782451.058,
  39             'duration': 58.5585,
  40             'tags': ['hornofafrica', 'explorations'],
  41         },
  42     }, {
  43         'url': 'https://gettr.com/post/p4iahp',
  44         'info_dict': {
  45             'id': 'p4iahp',
  46             'title': 'md5:b03c07883db6fbc1aab88877a6c3b149',
  47             'description': 'md5:741b7419d991c403196ed2ea7749a39d',
  48             'ext': 'mp4',
  49             'uploader': 'Neues Forum Freiheit',
  50             'uploader_id': 'nf_freiheit',
  51             'upload_date': '20210718',
  52             'thumbnail': r're:^https?://.+/out\.jpg',
  53             'timestamp': 1626594455.017,
  54             'duration': 23,
  55             'tags': 'count:12',
  56         },
  57     }, {
  58         # quote post
  59         'url': 'https://gettr.com/post/pxn5b743a9',
  60         'only_matching': True,
  61     }, {
  62         # quote with video
  63         'url': 'https://gettr.com/post/pxtiiz5ca2',
  64         'only_matching': True,
  65     }, {
  66         # streaming embed
  67         'url': 'https://gettr.com/post/pxlu8p3b13',
  68         'only_matching': True,
  69     }, {
  70         # youtube embed
  71         'url': 'https://gettr.com/post/pv6wp9e24c',
  72         'only_matching': True,
  73         'add_ie': ['Youtube'],
  74     }]
  75
  76     def _real_extract(self, url):
  77         post_id = self._match_id(url)
  78         webpage = self._download_webpage(url, post_id)
  79         api_data = self._call_api(f'post/{post_id}?incl="poststats|userinfo"', post_id)
  80
  81         post_data = api_data.get('data')
  82         user_data = try_get(api_data, lambda x: x['aux']['uinf'][post_data['uid']], dict) or {}
  83
  84         vid = post_data.get('vid')
  85         ovid = post_data.get('ovid')
  86
  87         if post_data.get('p_type') == 'stream':
  88             return self.url_result(f'https://gettr.com/streaming/{post_id}', ie='GettrStreaming', video_id=post_id)
  89
  90         if not (ovid or vid):
  91             embed_url = url_or_none(post_data.get('prevsrc'))
  92             shared_post_id = traverse_obj(api_data, ('aux', 'shrdpst', '_id'), ('data', 'rpstIds', 0), expected_type=str)
  93
  94             if embed_url:
  95                 return self.url_result(embed_url)
  96             elif shared_post_id:
  97                 return self.url_result(f'https://gettr.com/post/{shared_post_id}', ie='Gettr', video_id=shared_post_id)
  98             else:
  99                 raise ExtractorError('There\'s no video in this post.')
 100
 101         title = description = str_or_none(
 102             post_data.get('txt') or self._og_search_description(webpage))
 103
 104         uploader = str_or_none(
 105             user_data.get('nickname')
 106             or self._search_regex(r'^(.+?) on GETTR', self._og_search_title(webpage, default=''), 'uploader', fatal=False))
 107
 108         if uploader:
 109             title = f'{uploader} - {title}'
 110
 111         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 112             urljoin(self._MEDIA_BASE_URL, vid), post_id, 'mp4',
 113             entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if vid else ([], {})
 114
 115         if ovid:
 116             formats.append({
 117                 'url': urljoin(self._MEDIA_BASE_URL, ovid),
 118                 'format_id': 'ovid',
 119                 'ext': 'mp4',
 120                 'width': int_or_none(post_data.get('vid_wid')),
 121                 'height': int_or_none(post_data.get('vid_hgt')),
 122             })
 123
 124         return {
 125             'id': post_id,
 126             'title': title,
 127             'description': description,
 128             'formats': formats,
 129             'subtitles': subtitles,
 130             'uploader': uploader,
 131             'uploader_id': str_or_none(
 132                 dict_get(user_data, ['_id', 'username'])
 133                 or post_data.get('uid')),
 134             'thumbnail': url_or_none(
 135                 urljoin(self._MEDIA_BASE_URL, post_data.get('main'))
 136                 or self._html_search_meta(['og:image', 'image'], webpage, 'thumbnail', fatal=False)),
 137             'timestamp': float_or_none(dict_get(post_data, ['cdate', 'udate']), scale=1000),
 138             'duration': float_or_none(post_data.get('vid_dur')),
 139             'tags': post_data.get('htgs'),
 140         }
 141
 142
 143 class GettrStreamingIE(GettrBaseIE):
 144     _VALID_URL = GettrBaseIE._BASE_REGEX + r'streaming/(?P<id>[a-z0-9]+)'
 145
 146     _TESTS = [{
 147         'url': 'https://gettr.com/streaming/psoiulc122',
 148         'info_dict': {
 149             'id': 'psoiulc122',
 150             'ext': 'mp4',
 151             'description': 'md5:56bca4b8f48f1743d9fd03d49c723017',
 152             'view_count': int,
 153             'uploader': 'Corona Investigative Committee',
 154             'uploader_id': 'coronacommittee',
 155             'duration': 5180.184,
 156             'thumbnail': r're:^https?://.+',
 157             'title': 'Day 1: Opening Session of the Grand Jury Proceeding',
 158             'timestamp': 1644080997.164,
 159             'upload_date': '20220205',
 160         },
 161     }, {
 162         'url': 'https://gettr.com/streaming/psfmeefcc1',
 163         'info_dict': {
 164             'id': 'psfmeefcc1',
 165             'ext': 'mp4',
 166             'title': 'Session 90: "The Virus Of Power"',
 167             'view_count': int,
 168             'uploader_id': 'coronacommittee',
 169             'description': 'md5:98986acdf656aa836bf36f9c9704c65b',
 170             'uploader': 'Corona Investigative Committee',
 171             'thumbnail': r're:^https?://.+',
 172             'duration': 21872.507,
 173             'timestamp': 1643976662.858,
 174             'upload_date': '20220204',
 175         },
 176     }]
 177
 178     def _real_extract(self, url):
 179         video_id = self._match_id(url)
 180         video_info = self._call_api(f'live/join/{video_id}', video_id, data={})
 181
 182         live_info = video_info['broadcast']
 183         live_url = url_or_none(live_info.get('url'))
 184
 185         formats, subtitles = self._extract_m3u8_formats_and_subtitles(
 186             live_url, video_id, ext='mp4',
 187             entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else ([], {})
 188
 189         thumbnails = [{
 190             'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
 191         } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs'], list) or []]
 192
 193         return {
 194             'id': video_id,
 195             'title': try_get(video_info, lambda x: x['postData']['ttl'], str),
 196             'description': try_get(video_info, lambda x: x['postData']['dsc'], str),
 197             'formats': formats,
 198             'subtitles': subtitles,
 199             'thumbnails': thumbnails,
 200             'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname'], str),
 201             'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id'], str),
 202             'view_count': int_or_none(live_info.get('viewsCount')),
 203             'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
 204             'duration': float_or_none(live_info.get('duration'), scale=1000),
 205             'is_live': bool_or_none(live_info.get('isLive')),
 206         }