yt_dlp/extractor/mixcloud.py

   1 import base64
   2 import itertools
   3 import urllib.parse
   4
   5 from .common import InfoExtractor
   6 from ..compat import compat_ord
   7 from ..utils import (
   8     ExtractorError,
   9     int_or_none,
  10     parse_iso8601,
  11     strip_or_none,
  12     try_get,
  13 )
  14
  15
  16 class MixcloudBaseIE(InfoExtractor):
  17     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
  18         lookup_key = object_type + 'Lookup'
  19         return self._download_json(
  20             'https://app.mixcloud.com/graphql', display_id, query={
  21                 'query': '''{
  22   %s(lookup: {username: "%s"%s}) {
  23     %s
  24   }
  25 }''' % (lookup_key, username, f', slug: "{slug}"' if slug else '', object_fields),  # noqa: UP031
  26             })['data'][lookup_key]
  27
  28
  29 class MixcloudIE(MixcloudBaseIE):
  30     _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)'
  31     IE_NAME = 'mixcloud'
  32
  33     _TESTS = [{
  34         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
  35         'info_dict': {
  36             'id': 'dholbach_cryptkeeper',
  37             'ext': 'm4a',
  38             'title': 'Cryptkeeper',
  39             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
  40             'uploader': 'Daniel Holbach',
  41             'uploader_id': 'dholbach',
  42             'thumbnail': r're:https?://.*\.jpg',
  43             'view_count': int,
  44             'timestamp': 1321359578,
  45             'upload_date': '20111115',
  46             'uploader_url': 'https://www.mixcloud.com/dholbach/',
  47             'artist': 'Submorphics & Chino , Telekinesis, Porter Robinson, Enei, Breakage ft Jess Mills',
  48             'duration': 3723,
  49             'tags': [],
  50             'comment_count': int,
  51             'repost_count': int,
  52             'like_count': int,
  53         },
  54         'params': {'skip_download': 'm3u8'},
  55     }, {
  56         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
  57         'info_dict': {
  58             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
  59             'ext': 'mp3',
  60             'title': 'Caribou 7 inch Vinyl Mix & Chat',
  61             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
  62             'uploader': 'Gilles Peterson Worldwide',
  63             'uploader_id': 'gillespeterson',
  64             'thumbnail': 're:https?://.*',
  65             'view_count': int,
  66             'timestamp': 1422987057,
  67             'upload_date': '20150203',
  68             'uploader_url': 'https://www.mixcloud.com/gillespeterson/',
  69             'duration': 2992,
  70             'tags': [],
  71             'comment_count': int,
  72             'repost_count': int,
  73             'like_count': int,
  74         },
  75         'params': {'skip_download': '404 playback error on site'},
  76     }, {
  77         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
  78         'only_matching': True,
  79     }]
  80     _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD'
  81
  82     @staticmethod
  83     def _decrypt_xor_cipher(key, ciphertext):
  84         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
  85         return ''.join([
  86             chr(compat_ord(ch) ^ compat_ord(k))
  87             for ch, k in zip(ciphertext, itertools.cycle(key))])
  88
  89     def _real_extract(self, url):
  90         username, slug = self._match_valid_url(url).groups()
  91         username, slug = urllib.parse.unquote(username), urllib.parse.unquote(slug)
  92         track_id = f'{username}_{slug}'
  93
  94         cloudcast = self._call_api('cloudcast', '''audioLength
  95     comments(first: 100) {
  96       edges {
  97         node {
  98           comment
  99           created
 100           user {
 101             displayName
 102             username
 103           }
 104         }
 105       }
 106       totalCount
 107     }
 108     description
 109     favorites {
 110       totalCount
 111     }
 112     featuringArtistList
 113     isExclusive
 114     name
 115     owner {
 116       displayName
 117       url
 118       username
 119     }
 120     picture(width: 1024, height: 1024) {
 121         url
 122     }
 123     plays
 124     publishDate
 125     reposts {
 126       totalCount
 127     }
 128     streamInfo {
 129       dashUrl
 130       hlsUrl
 131       url
 132     }
 133     tags {
 134       tag {
 135         name
 136       }
 137     }
 138     restrictedReason
 139     id''', track_id, username, slug)
 140
 141         if not cloudcast:
 142             raise ExtractorError('Track not found', expected=True)
 143
 144         reason = cloudcast.get('restrictedReason')
 145         if reason == 'tracklist':
 146             raise ExtractorError('Track unavailable in your country due to licensing restrictions', expected=True)
 147         elif reason == 'repeat_play':
 148             raise ExtractorError('You have reached your play limit for this track', expected=True)
 149         elif reason:
 150             raise ExtractorError('Track is restricted', expected=True)
 151
 152         title = cloudcast['name']
 153
 154         stream_info = cloudcast['streamInfo']
 155         formats = []
 156
 157         for url_key in ('url', 'hlsUrl', 'dashUrl'):
 158             format_url = stream_info.get(url_key)
 159             if not format_url:
 160                 continue
 161             decrypted = self._decrypt_xor_cipher(
 162                 self._DECRYPTION_KEY, base64.b64decode(format_url))
 163             if url_key == 'hlsUrl':
 164                 formats.extend(self._extract_m3u8_formats(
 165                     decrypted, track_id, 'mp4', entry_protocol='m3u8_native',
 166                     m3u8_id='hls', fatal=False))
 167             elif url_key == 'dashUrl':
 168                 formats.extend(self._extract_mpd_formats(
 169                     decrypted, track_id, mpd_id='dash', fatal=False))
 170             else:
 171                 formats.append({
 172                     'format_id': 'http',
 173                     'url': decrypted,
 174                     'vcodec': 'none',
 175                     'downloader_options': {
 176                         # Mixcloud starts throttling at >~5M
 177                         'http_chunk_size': 5242880,
 178                     },
 179                 })
 180
 181         if not formats and cloudcast.get('isExclusive'):
 182             self.raise_login_required(metadata_available=True)
 183
 184         comments = []
 185         for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []):
 186             node = edge.get('node') or {}
 187             text = strip_or_none(node.get('comment'))
 188             if not text:
 189                 continue
 190             user = node.get('user') or {}
 191             comments.append({
 192                 'author': user.get('displayName'),
 193                 'author_id': user.get('username'),
 194                 'text': text,
 195                 'timestamp': parse_iso8601(node.get('created')),
 196             })
 197
 198         tags = []
 199         for t in cloudcast.get('tags'):
 200             tag = try_get(t, lambda x: x['tag']['name'], str)
 201             if not tag:
 202                 tags.append(tag)
 203
 204         get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount']))
 205
 206         owner = cloudcast.get('owner') or {}
 207
 208         return {
 209             'id': track_id,
 210             'title': title,
 211             'formats': formats,
 212             'description': cloudcast.get('description'),
 213             'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], str),
 214             'uploader': owner.get('displayName'),
 215             'timestamp': parse_iso8601(cloudcast.get('publishDate')),
 216             'uploader_id': owner.get('username'),
 217             'uploader_url': owner.get('url'),
 218             'duration': int_or_none(cloudcast.get('audioLength')),
 219             'view_count': int_or_none(cloudcast.get('plays')),
 220             'like_count': get_count('favorites'),
 221             'repost_count': get_count('reposts'),
 222             'comment_count': get_count('comments'),
 223             'comments': comments,
 224             'tags': tags,
 225             'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None,
 226         }
 227
 228
 229 class MixcloudPlaylistBaseIE(MixcloudBaseIE):
 230     def _get_cloudcast(self, node):
 231         return node
 232
 233     def _get_playlist_title(self, title, slug):
 234         return title
 235
 236     def _real_extract(self, url):
 237         username, slug = self._match_valid_url(url).groups()
 238         username = urllib.parse.unquote(username)
 239         if not slug:
 240             slug = 'uploads'
 241         else:
 242             slug = urllib.parse.unquote(slug)
 243         playlist_id = f'{username}_{slug}'
 244
 245         is_playlist_type = self._ROOT_TYPE == 'playlist'
 246         playlist_type = 'items' if is_playlist_type else slug
 247         list_filter = ''
 248
 249         has_next_page = True
 250         entries = []
 251         while has_next_page:
 252             playlist = self._call_api(
 253                 self._ROOT_TYPE, '''%s
 254     %s
 255     %s(first: 100%s) {
 256       edges {
 257         node {
 258           %s
 259         }
 260       }
 261       pageInfo {
 262         endCursor
 263         hasNextPage
 264       }
 265     }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE),  # noqa: UP031
 266                 playlist_id, username, slug if is_playlist_type else None)
 267
 268             items = playlist.get(playlist_type) or {}
 269             for edge in items.get('edges', []):
 270                 cloudcast = self._get_cloudcast(edge.get('node') or {})
 271                 cloudcast_url = cloudcast.get('url')
 272                 if not cloudcast_url:
 273                     continue
 274                 item_slug = try_get(cloudcast, lambda x: x['slug'], str)
 275                 owner_username = try_get(cloudcast, lambda x: x['owner']['username'], str)
 276                 video_id = f'{owner_username}_{item_slug}' if item_slug and owner_username else None
 277                 entries.append(self.url_result(
 278                     cloudcast_url, MixcloudIE.ie_key(), video_id))
 279
 280             page_info = items['pageInfo']
 281             has_next_page = page_info['hasNextPage']
 282             list_filter = ', after: "{}"'.format(page_info['endCursor'])
 283
 284         return self.playlist_result(
 285             entries, playlist_id,
 286             self._get_playlist_title(playlist[self._TITLE_KEY], slug),
 287             playlist.get(self._DESCRIPTION_KEY))
 288
 289
 290 class MixcloudUserIE(MixcloudPlaylistBaseIE):
 291     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<id>[^/]+)/(?P<type>uploads|favorites|listens|stream)?/?$'
 292     IE_NAME = 'mixcloud:user'
 293
 294     _TESTS = [{
 295         'url': 'http://www.mixcloud.com/dholbach/',
 296         'info_dict': {
 297             'id': 'dholbach_uploads',
 298             'title': 'Daniel Holbach (uploads)',
 299             'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
 300         },
 301         'playlist_mincount': 36,
 302     }, {
 303         'url': 'http://www.mixcloud.com/dholbach/uploads/',
 304         'info_dict': {
 305             'id': 'dholbach_uploads',
 306             'title': 'Daniel Holbach (uploads)',
 307             'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
 308         },
 309         'playlist_mincount': 36,
 310     }, {
 311         'url': 'http://www.mixcloud.com/dholbach/favorites/',
 312         'info_dict': {
 313             'id': 'dholbach_favorites',
 314             'title': 'Daniel Holbach (favorites)',
 315             'description': 'md5:a3f468a60ac8c3e1f8616380fc469b2b',
 316         },
 317         # 'params': {
 318         #     'playlist_items': '1-100',
 319         # },
 320         'playlist_mincount': 396,
 321     }, {
 322         'url': 'http://www.mixcloud.com/dholbach/listens/',
 323         'info_dict': {
 324             'id': 'dholbach_listens',
 325             'title': 'Daniel Holbach (listens)',
 326             'description': 'md5:b60d776f0bab534c5dabe0a34e47a789',
 327         },
 328         # 'params': {
 329         #     'playlist_items': '1-100',
 330         # },
 331         'playlist_mincount': 1623,
 332         'skip': 'Large list',
 333     }, {
 334         'url': 'https://www.mixcloud.com/FirstEar/stream/',
 335         'info_dict': {
 336             'id': 'FirstEar_stream',
 337             'title': 'First Ear (stream)',
 338             'description': 'we maraud for ears',
 339         },
 340         'playlist_mincount': 269,
 341     }]
 342
 343     _TITLE_KEY = 'displayName'
 344     _DESCRIPTION_KEY = 'biog'
 345     _ROOT_TYPE = 'user'
 346     _NODE_TEMPLATE = '''slug
 347           url
 348           owner { username }'''
 349
 350     def _get_playlist_title(self, title, slug):
 351         return f'{title} ({slug})'
 352
 353
 354 class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
 355     _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P<user>[^/]+)/playlists/(?P<playlist>[^/]+)/?$'
 356     IE_NAME = 'mixcloud:playlist'
 357
 358     _TESTS = [{
 359         'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/',
 360         'info_dict': {
 361             'id': 'maxvibes_jazzcat-on-ness-radio',
 362             'title': 'Ness Radio sessions',
 363         },
 364         'playlist_mincount': 59,
 365     }]
 366     _TITLE_KEY = 'name'
 367     _DESCRIPTION_KEY = 'description'
 368     _ROOT_TYPE = 'playlist'
 369     _NODE_TEMPLATE = '''cloudcast {
 370             slug
 371             url
 372             owner { username }
 373           }'''
 374
 375     def _get_cloudcast(self, node):
 376         return node.get('cloudcast') or {}