yt_dlp/extractor/imgur.py

   1 import functools
   2 import re
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     ExtractorError,
   7     determine_ext,
   8     float_or_none,
   9     int_or_none,
  10     js_to_json,
  11     mimetype2ext,
  12     parse_iso8601,
  13     str_or_none,
  14     strip_or_none,
  15     traverse_obj,
  16     url_or_none,
  17 )
  18
  19
  20 class ImgurBaseIE(InfoExtractor):
  21     _CLIENT_ID = '546c25a59c58ad7'
  22
  23     @classmethod
  24     def _imgur_result(cls, item_id):
  25         return cls.url_result(f'https://imgur.com/{item_id}', ImgurIE, item_id)
  26
  27     def _call_api(self, endpoint, video_id, **kwargs):
  28         return self._download_json(
  29             f'https://api.imgur.com/post/v1/{endpoint}/{video_id}?client_id={self._CLIENT_ID}&include=media,account',
  30             video_id, **kwargs)
  31
  32     @staticmethod
  33     def get_description(s):
  34         if 'Discover the magic of the internet at Imgur' in s:
  35             return None
  36         return s or None
  37
  38
  39 class ImgurIE(ImgurBaseIE):
  40     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)(?:[^/?#]+-)?(?P<id>[a-zA-Z0-9]+)'
  41
  42     _TESTS = [{
  43         'url': 'https://imgur.com/A61SaA1',
  44         'info_dict': {
  45             'id': 'A61SaA1',
  46             'ext': 'mp4',
  47             'title': 'MRW gifv is up and running without any bugs',
  48             'timestamp': 1416446068,
  49             'upload_date': '20141120',
  50             'dislike_count': int,
  51             'comment_count': int,
  52             'release_timestamp': 1416446068,
  53             'release_date': '20141120',
  54             'like_count': int,
  55             'thumbnail': 'https://i.imgur.com/A61SaA1h.jpg',
  56         },
  57     }, {
  58         # Test with URL slug
  59         'url': 'https://imgur.com/mrw-gifv-is-up-running-without-any-bugs-A61SaA1',
  60         'info_dict': {
  61             'id': 'A61SaA1',
  62             'ext': 'mp4',
  63             'title': 'MRW gifv is up and running without any bugs',
  64             'timestamp': 1416446068,
  65             'upload_date': '20141120',
  66             'dislike_count': int,
  67             'comment_count': int,
  68             'release_timestamp': 1416446068,
  69             'release_date': '20141120',
  70             'like_count': int,
  71             'thumbnail': 'https://i.imgur.com/A61SaA1h.jpg',
  72         },
  73     }, {
  74         'url': 'https://i.imgur.com/A61SaA1.gifv',
  75         'only_matching': True,
  76     }, {
  77         'url': 'https://i.imgur.com/crGpqCV.mp4',
  78         'only_matching': True,
  79     }, {
  80         'url': 'https://i.imgur.com/jxBXAMC.gifv',
  81         'info_dict': {
  82             'id': 'jxBXAMC',
  83             'ext': 'mp4',
  84             'title': 'Fahaka puffer feeding',
  85             'timestamp': 1533835503,
  86             'upload_date': '20180809',
  87             'release_date': '20180809',
  88             'like_count': int,
  89             'duration': 30.0,
  90             'comment_count': int,
  91             'release_timestamp': 1533835503,
  92             'thumbnail': 'https://i.imgur.com/jxBXAMCh.jpg',
  93             'dislike_count': int,
  94         },
  95     }, {
  96         # needs Accept header, ref: https://github.com/yt-dlp/yt-dlp/issues/9458
  97         'url': 'https://imgur.com/zV03bd5',
  98         'md5': '59df97884e8ba76143ff6b640a0e2904',
  99         'info_dict': {
 100             'id': 'zV03bd5',
 101             'ext': 'mp4',
 102             'title': 'Ive - Liz',
 103             'timestamp': 1710491255,
 104             'upload_date': '20240315',
 105             'like_count': int,
 106             'dislike_count': int,
 107             'duration': 56.92,
 108             'comment_count': int,
 109             'release_timestamp': 1710491255,
 110             'release_date': '20240315',
 111             'thumbnail': 'https://i.imgur.com/zV03bd5h.jpg',
 112         },
 113     }]
 114
 115     def _real_extract(self, url):
 116         video_id = self._match_id(url)
 117         data = self._call_api('media', video_id)
 118         if not traverse_obj(data, ('media', 0, (
 119                 ('type', {lambda t: t == 'video' or None}),
 120                 ('metadata', 'is_animated'))), get_all=False):
 121             raise ExtractorError(f'{video_id} is not a video or animated image', expected=True)
 122         webpage = self._download_webpage(
 123             f'https://i.imgur.com/{video_id}.gifv', video_id, fatal=False) or ''
 124         formats = []
 125
 126         media_fmt = traverse_obj(data, ('media', 0, {
 127             'url': ('url', {url_or_none}),
 128             'ext': ('ext', {str}),
 129             'width': ('width', {int_or_none}),
 130             'height': ('height', {int_or_none}),
 131             'filesize': ('size', {int_or_none}),
 132             'acodec': ('metadata', 'has_sound', {lambda b: None if b else 'none'}),
 133         }))
 134         media_url = media_fmt.get('url')
 135         if media_url:
 136             if not media_fmt.get('ext'):
 137                 media_fmt['ext'] = mimetype2ext(traverse_obj(
 138                     data, ('media', 0, 'mime_type'))) or determine_ext(media_url)
 139             if traverse_obj(data, ('media', 0, 'type')) == 'image':
 140                 media_fmt['acodec'] = 'none'
 141                 media_fmt.setdefault('preference', -10)
 142             formats.append(media_fmt)
 143
 144         video_elements = self._search_regex(
 145             r'(?s)<div class="video-elements">(.*?)</div>',
 146             webpage, 'video elements', default=None)
 147
 148         if video_elements:
 149             def og_get_size(media_type):
 150                 return {
 151                     p: int_or_none(self._og_search_property(f'{media_type}:{p}', webpage, default=None))
 152                     for p in ('width', 'height')
 153                 }
 154
 155             size = og_get_size('video')
 156             if not any(size.values()):
 157                 size = og_get_size('image')
 158
 159             formats = traverse_obj(
 160                 re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements),
 161                 (..., {
 162                     'format_id': ('type', {lambda s: s.partition('/')[2]}),
 163                     'url': ('src', {self._proto_relative_url}),
 164                     'ext': ('type', {mimetype2ext}),
 165                 }))
 166             for f in formats:
 167                 f.update(size)
 168
 169             # We can get the original gif format from the webpage as well
 170             gif_json = traverse_obj(self._search_json(
 171                 r'var\s+videoItem\s*=', webpage, 'GIF info', video_id,
 172                 transform_source=js_to_json, fatal=False), {
 173                     'url': ('gifUrl', {self._proto_relative_url}),
 174                     'filesize': ('size', {int_or_none}),
 175             })
 176             if gif_json:
 177                 gif_json.update(size)
 178                 gif_json.update({
 179                     'format_id': 'gif',
 180                     'preference': -10,  # gifs < videos
 181                     'ext': 'gif',
 182                     'acodec': 'none',
 183                     'vcodec': 'gif',
 184                     'container': 'gif',
 185                 })
 186                 formats.append(gif_json)
 187
 188         search = functools.partial(self._html_search_meta, html=webpage, default=None)
 189
 190         twitter_fmt = {
 191             'format_id': 'twitter',
 192             'url': url_or_none(search('twitter:player:stream')),
 193             'ext': mimetype2ext(search('twitter:player:stream:content_type')),
 194             'width': int_or_none(search('twitter:width')),
 195             'height': int_or_none(search('twitter:height')),
 196         }
 197         if twitter_fmt['url']:
 198             formats.append(twitter_fmt)
 199
 200         if not formats:
 201             self.raise_no_formats(
 202                 f'No sources found for video {video_id}. Maybe a plain image?', expected=True)
 203         self._remove_duplicate_formats(formats)
 204
 205         return {
 206             'title': self._og_search_title(webpage, default=None),
 207             'description': self.get_description(self._og_search_description(webpage, default='')),
 208             **traverse_obj(data, {
 209                 'uploader_id': ('account_id', {lambda a: str(a) if int_or_none(a) else None}),
 210                 'uploader': ('account', 'username', {lambda x: strip_or_none(x) or None}),
 211                 'uploader_url': ('account', 'avatar_url', {url_or_none}),
 212                 'like_count': ('upvote_count', {int_or_none}),
 213                 'dislike_count': ('downvote_count', {int_or_none}),
 214                 'comment_count': ('comment_count', {int_or_none}),
 215                 'age_limit': ('is_mature', {lambda x: 18 if x else None}),
 216                 'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
 217                 'release_timestamp': ('created_at', {parse_iso8601}),
 218             }, get_all=False),
 219             **traverse_obj(data, ('media', 0, 'metadata', {
 220                 'title': ('title', {lambda x: strip_or_none(x) or None}),
 221                 'description': ('description', {self.get_description}),
 222                 'duration': ('duration', {float_or_none}),
 223                 'timestamp': (('updated_at', 'created_at'), {parse_iso8601}),
 224                 'release_timestamp': ('created_at', {parse_iso8601}),
 225             }), get_all=False),
 226             'id': video_id,
 227             'formats': formats,
 228             'thumbnails': [{
 229                 'url': thumbnail_url,
 230                 'http_headers': {'Accept': '*/*'},
 231             }] if (thumbnail_url := search(['thumbnailUrl', 'twitter:image', 'og:image'])) else None,
 232             'http_headers': {'Accept': '*/*'},
 233         }
 234
 235
 236 class ImgurGalleryBaseIE(ImgurBaseIE):
 237     _GALLERY = True
 238
 239     def _real_extract(self, url):
 240         gallery_id = self._match_id(url)
 241
 242         data = self._call_api('albums', gallery_id, fatal=False, expected_status=404)
 243
 244         info = traverse_obj(data, {
 245             'title': ('title', {lambda x: strip_or_none(x) or None}),
 246             'description': ('description', {self.get_description}),
 247         })
 248
 249         if traverse_obj(data, 'is_album'):
 250
 251             items = traverse_obj(data, (
 252                 'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'],
 253                 'id', {lambda x: str_or_none(x) or None}))
 254
 255             # if a gallery with exactly one video, apply album metadata to video
 256             media_id = None
 257             if self._GALLERY and len(items) == 1:
 258                 media_id = items[0]
 259
 260             if not media_id:
 261                 result = self.playlist_result(
 262                     map(self._imgur_result, items), gallery_id)
 263                 result.update(info)
 264                 return result
 265             gallery_id = media_id
 266
 267         result = self._imgur_result(gallery_id)
 268         info['_type'] = 'url_transparent'
 269         result.update(info)
 270         return result
 271
 272
 273 class ImgurGalleryIE(ImgurGalleryBaseIE):
 274     IE_NAME = 'imgur:gallery'
 275     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/?#]+)/(?:[^/?#]+-)?(?P<id>[a-zA-Z0-9]+)'
 276
 277     _TESTS = [{
 278         # TODO: static images - replace with animated/video gallery
 279         'url': 'http://imgur.com/topic/Aww/ll5Vk',
 280         'only_matching': True,
 281     }, {
 282         'url': 'https://imgur.com/gallery/YcAQlkx',
 283         'add_ies': ['Imgur'],
 284         'info_dict': {
 285             'id': 'YcAQlkx',
 286             'ext': 'mp4',
 287             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
 288             'timestamp': 1358554297,
 289             'upload_date': '20130119',
 290             'uploader_id': '1648642',
 291             'uploader': 'wittyusernamehere',
 292             'release_timestamp': 1358554297,
 293             'thumbnail': 'https://i.imgur.com/YcAQlkxh.jpg',
 294             'release_date': '20130119',
 295             'uploader_url': 'https://i.imgur.com/N5Flb2v_d.png?maxwidth=290&fidelity=grand',
 296             'comment_count': int,
 297             'dislike_count': int,
 298             'like_count': int,
 299         },
 300     }, {
 301         # Test with slug
 302         'url': 'https://imgur.com/gallery/classic-steve-carell-gif-cracks-me-up-everytime-repost-downvotes-YcAQlkx',
 303         'add_ies': ['Imgur'],
 304         'info_dict': {
 305             'id': 'YcAQlkx',
 306             'ext': 'mp4',
 307             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
 308             'timestamp': 1358554297,
 309             'upload_date': '20130119',
 310             'uploader_id': '1648642',
 311             'uploader': 'wittyusernamehere',
 312             'release_timestamp': 1358554297,
 313             'release_date': '20130119',
 314             'thumbnail': 'https://i.imgur.com/YcAQlkxh.jpg',
 315             'uploader_url': 'https://i.imgur.com/N5Flb2v_d.png?maxwidth=290&fidelity=grand',
 316             'comment_count': int,
 317             'dislike_count': int,
 318             'like_count': int,
 319         },
 320     }, {
 321         # TODO: static image - replace with animated/video gallery
 322         'url': 'http://imgur.com/topic/Funny/N8rOudd',
 323         'only_matching': True,
 324     }, {
 325         'url': 'http://imgur.com/r/aww/VQcQPhM',
 326         'add_ies': ['Imgur'],
 327         'info_dict': {
 328             'id': 'VQcQPhM',
 329             'ext': 'mp4',
 330             'title': 'The boss is here',
 331             'timestamp': 1476494751,
 332             'upload_date': '20161015',
 333             'uploader_id': '19138530',
 334             'uploader': 'thematrixcam',
 335             'comment_count': int,
 336             'dislike_count': int,
 337             'uploader_url': 'https://i.imgur.com/qCjr5Pi_d.png?maxwidth=290&fidelity=grand',
 338             'release_timestamp': 1476494751,
 339             'like_count': int,
 340             'release_date': '20161015',
 341             'thumbnail': 'https://i.imgur.com/VQcQPhMh.jpg',
 342         },
 343     },
 344         # from https://github.com/ytdl-org/youtube-dl/pull/16674
 345         {
 346         'url': 'https://imgur.com/t/unmuted/6lAn9VQ',
 347         'info_dict': {
 348             'id': '6lAn9VQ',
 349             'title': 'Penguins !',
 350         },
 351         'playlist_count': 3,
 352     }, {
 353         'url': 'https://imgur.com/t/unmuted/penguins-penguins-6lAn9VQ',
 354         'info_dict': {
 355             'id': '6lAn9VQ',
 356             'title': 'Penguins !',
 357         },
 358         'playlist_count': 3,
 359     }, {
 360         'url': 'https://imgur.com/t/unmuted/kx2uD3C',
 361         'add_ies': ['Imgur'],
 362         'info_dict': {
 363             'id': 'ZVMv45i',
 364             'ext': 'mp4',
 365             'title': 'Intruder',
 366             'timestamp': 1528129683,
 367             'upload_date': '20180604',
 368             'release_timestamp': 1528129683,
 369             'release_date': '20180604',
 370             'like_count': int,
 371             'dislike_count': int,
 372             'comment_count': int,
 373             'duration': 30.03,
 374             'thumbnail': 'https://i.imgur.com/ZVMv45ih.jpg',
 375         },
 376     }, {
 377         'url': 'https://imgur.com/t/unmuted/wXSK0YH',
 378         'add_ies': ['Imgur'],
 379         'info_dict': {
 380             'id': 'JCAP4io',
 381             'ext': 'mp4',
 382             'title': 're:I got the blues$',
 383             'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.',
 384             'timestamp': 1527809525,
 385             'upload_date': '20180531',
 386             'like_count': int,
 387             'dislike_count': int,
 388             'duration': 30.03,
 389             'comment_count': int,
 390             'release_timestamp': 1527809525,
 391             'thumbnail': 'https://i.imgur.com/JCAP4ioh.jpg',
 392             'release_date': '20180531',
 393         },
 394     }]
 395
 396
 397 class ImgurAlbumIE(ImgurGalleryBaseIE):
 398     IE_NAME = 'imgur:album'
 399     _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?:[^/?#]+-)?(?P<id>[a-zA-Z0-9]+)'
 400     _GALLERY = False
 401     _TESTS = [{
 402         # TODO: only static images - replace with animated/video gallery
 403         'url': 'http://imgur.com/a/j6Orj',
 404         'only_matching': True,
 405     },
 406         # from https://github.com/ytdl-org/youtube-dl/pull/21693
 407         {
 408         'url': 'https://imgur.com/a/iX265HX',
 409         'info_dict': {
 410             'id': 'iX265HX',
 411             'title': 'enen-no-shouboutai',
 412         },
 413         'playlist_count': 2,
 414     }, {
 415         # Test with URL slug
 416         'url': 'https://imgur.com/a/enen-no-shouboutai-iX265HX',
 417         'info_dict': {
 418             'id': 'iX265HX',
 419             'title': 'enen-no-shouboutai',
 420         },
 421         'playlist_count': 2,
 422     }, {
 423         'url': 'https://imgur.com/a/8pih2Ed',
 424         'info_dict': {
 425             'id': '8pih2Ed',
 426         },
 427         'playlist_mincount': 1,
 428     }]