yt_dlp/extractor/minds.py

   1 from .common import InfoExtractor
   2 from ..utils import (
   3     clean_html,
   4     format_field,
   5     int_or_none,
   6     str_or_none,
   7     strip_or_none,
   8 )
   9
  10
  11 class MindsBaseIE(InfoExtractor):
  12     _VALID_URL_BASE = r'https?://(?:www\.)?minds\.com/'
  13
  14     def _call_api(self, path, video_id, resource, query=None):
  15         api_url = 'https://www.minds.com/api/' + path
  16         token = self._get_cookies(api_url).get('XSRF-TOKEN')
  17         return self._download_json(
  18             api_url, video_id, f'Downloading {resource} JSON metadata', headers={
  19                 'Referer': 'https://www.minds.com/',
  20                 'X-XSRF-TOKEN': token.value if token else '',
  21             }, query=query)
  22
  23
  24 class MindsIE(MindsBaseIE):
  25     IE_NAME = 'minds'
  26     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?:media|newsfeed|archive/view)/(?P<id>[0-9]+)'
  27     _TESTS = [{
  28         'url': 'https://www.minds.com/media/100000000000086822',
  29         'md5': '215a658184a419764852239d4970b045',
  30         'info_dict': {
  31             'id': '100000000000086822',
  32             'ext': 'mp4',
  33             'title': 'Minds intro sequence',
  34             'thumbnail': r're:https?://.+\.png',
  35             'uploader_id': 'ottman',
  36             'upload_date': '20130524',
  37             'timestamp': 1369404826,
  38             'uploader': 'Bill Ottman',
  39             'view_count': int,
  40             'like_count': int,
  41             'dislike_count': int,
  42             'tags': ['animation'],
  43             'comment_count': int,
  44             'license': 'attribution-cc',
  45         },
  46     }, {
  47         # entity.type == 'activity' and empty title
  48         'url': 'https://www.minds.com/newsfeed/798025111988506624',
  49         'md5': 'b2733a74af78d7fd3f541c4cbbaa5950',
  50         'info_dict': {
  51             'id': '798022190320226304',
  52             'ext': 'mp4',
  53             'title': '798022190320226304',
  54             'uploader': 'ColinFlaherty',
  55             'upload_date': '20180111',
  56             'timestamp': 1515639316,
  57             'uploader_id': 'ColinFlaherty',
  58         },
  59     }, {
  60         'url': 'https://www.minds.com/archive/view/715172106794442752',
  61         'only_matching': True,
  62     }, {
  63         # youtube perma_url
  64         'url': 'https://www.minds.com/newsfeed/1197131838022602752',
  65         'only_matching': True,
  66     }]
  67
  68     def _real_extract(self, url):
  69         entity_id = self._match_id(url)
  70         entity = self._call_api(
  71             'v1/entities/entity/' + entity_id, entity_id, 'entity')['entity']
  72         if entity.get('type') == 'activity':
  73             if entity.get('custom_type') == 'video':
  74                 video_id = entity['entity_guid']
  75             else:
  76                 return self.url_result(entity['perma_url'])
  77         else:
  78             assert entity['subtype'] == 'video'
  79             video_id = entity_id
  80         # 1080p and webm formats available only on the sources array
  81         video = self._call_api(
  82             'v2/media/video/' + video_id, video_id, 'video')
  83
  84         formats = []
  85         for source in (video.get('sources') or []):
  86             src = source.get('src')
  87             if not src:
  88                 continue
  89             formats.append({
  90                 'format_id': source.get('label'),
  91                 'height': int_or_none(source.get('size')),
  92                 'url': src,
  93             })
  94
  95         entity = video.get('entity') or entity
  96         owner = entity.get('ownerObj') or {}
  97         uploader_id = owner.get('username')
  98
  99         tags = entity.get('tags')
 100         if tags and isinstance(tags, str):
 101             tags = [tags]
 102
 103         thumbnail = None
 104         poster = video.get('poster') or entity.get('thumbnail_src')
 105         if poster:
 106             urlh = self._request_webpage(poster, video_id, fatal=False)
 107             if urlh:
 108                 thumbnail = urlh.url
 109
 110         return {
 111             'id': video_id,
 112             'title': entity.get('title') or video_id,
 113             'formats': formats,
 114             'description': clean_html(entity.get('description')) or None,
 115             'license': str_or_none(entity.get('license')),
 116             'timestamp': int_or_none(entity.get('time_created')),
 117             'uploader': strip_or_none(owner.get('name')),
 118             'uploader_id': uploader_id,
 119             'uploader_url': format_field(uploader_id, None, 'https://www.minds.com/%s'),
 120             'view_count': int_or_none(entity.get('play:count')),
 121             'like_count': int_or_none(entity.get('thumbs:up:count')),
 122             'dislike_count': int_or_none(entity.get('thumbs:down:count')),
 123             'tags': tags,
 124             'comment_count': int_or_none(entity.get('comments:count')),
 125             'thumbnail': thumbnail,
 126         }
 127
 128
 129 class MindsFeedBaseIE(MindsBaseIE):
 130     _PAGE_SIZE = 150
 131
 132     def _entries(self, feed_id):
 133         query = {'limit': self._PAGE_SIZE, 'sync': 1}
 134         i = 1
 135         while True:
 136             data = self._call_api(
 137                 f'v2/feeds/container/{feed_id}/videos',
 138                 feed_id, f'page {i}', query)
 139             entities = data.get('entities') or []
 140             for entity in entities:
 141                 guid = entity.get('guid')
 142                 if not guid:
 143                     continue
 144                 yield self.url_result(
 145                     'https://www.minds.com/newsfeed/' + guid,
 146                     MindsIE.ie_key(), guid)
 147             query['from_timestamp'] = data['load-next']
 148             if not (query['from_timestamp'] and len(entities) == self._PAGE_SIZE):
 149                 break
 150             i += 1
 151
 152     def _real_extract(self, url):
 153         feed_id = self._match_id(url)
 154         feed = self._call_api(
 155             f'v1/{self._FEED_PATH}/{feed_id}',
 156             feed_id, self._FEED_TYPE)[self._FEED_TYPE]
 157
 158         return self.playlist_result(
 159             self._entries(feed['guid']), feed_id,
 160             strip_or_none(feed.get('name')),
 161             feed.get('briefdescription'))
 162
 163
 164 class MindsChannelIE(MindsFeedBaseIE):
 165     _FEED_TYPE = 'channel'
 166     IE_NAME = 'minds:' + _FEED_TYPE
 167     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?!(?:newsfeed|media|api|archive|groups)/)(?P<id>[^/?&#]+)'
 168     _FEED_PATH = 'channel'
 169     _TEST = {
 170         'url': 'https://www.minds.com/ottman',
 171         'info_dict': {
 172             'id': 'ottman',
 173             'title': 'Bill Ottman',
 174             'description': 'Co-creator & CEO @minds',
 175         },
 176         'playlist_mincount': 54,
 177     }
 178
 179
 180 class MindsGroupIE(MindsFeedBaseIE):
 181     _FEED_TYPE = 'group'
 182     IE_NAME = 'minds:' + _FEED_TYPE
 183     _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'groups/profile/(?P<id>[0-9]+)'
 184     _FEED_PATH = 'groups/group'
 185     _TEST = {
 186         'url': 'https://www.minds.com/groups/profile/785582576369672204/feed/videos',
 187         'info_dict': {
 188             'id': '785582576369672204',
 189             'title': 'Cooking Videos',
 190         },
 191         'playlist_mincount': 1,
 192     }