yt_dlp/extractor/murrtube.py

   1 import functools
   2 import json
   3
   4 from .common import InfoExtractor
   5 from ..utils import (
   6     ExtractorError,
   7     OnDemandPagedList,
   8     clean_html,
   9     extract_attributes,
  10     get_element_by_class,
  11     get_element_html_by_id,
  12     parse_count,
  13     remove_end,
  14     update_url,
  15     urlencode_postdata,
  16 )
  17
  18
  19 class MurrtubeIE(InfoExtractor):
  20     _VALID_URL = r'''(?x)
  21                         (?:
  22                             murrtube:|
  23                             https?://murrtube\.net/(?:v/|videos/(?P<slug>[a-z0-9-]+?)-)
  24                         )
  25                         (?P<id>[A-Z0-9]{4}|[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})
  26                     '''
  27     _TESTS = [{
  28         'url': 'https://murrtube.net/videos/inferno-x-skyler-148b6f2a-fdcc-4902-affe-9c0f41aaaca0',
  29         'md5': '70380878a77e8565d4aea7f68b8bbb35',
  30         'info_dict': {
  31             'id': 'ca885d8456b95de529b6723b158032e11115d',
  32             'ext': 'mp4',
  33             'title': 'Inferno X Skyler',
  34             'description': 'Humping a very good slutty sheppy (roomate)',
  35             'uploader': 'Inferno Wolf',
  36             'age_limit': 18,
  37             'thumbnail': 'https://storage.murrtube.net/murrtube-production/ekbs3zcfvuynnqfx72nn2tkokvsd',
  38             'comment_count': int,
  39             'view_count': int,
  40             'like_count': int,
  41         },
  42     }, {
  43         'url': 'https://murrtube.net/v/0J2Q',
  44         'md5': '31262f6ac56f0ca75e5a54a0f3fefcb6',
  45         'info_dict': {
  46             'id': '8442998c52134968d9caa36e473e1a6bac6ca',
  47             'ext': 'mp4',
  48             'uploader': 'Hayel',
  49             'title': 'Who\'s in charge now?',
  50             'description': 'md5:795791e97e5b0f1805ea84573f02a997',
  51             'age_limit': 18,
  52             'thumbnail': 'https://storage.murrtube.net/murrtube-production/fb1ojjwiucufp34ya6hxu5vfqi5s',
  53             'comment_count': int,
  54             'view_count': int,
  55             'like_count': int,
  56         },
  57     }]
  58
  59     def _extract_count(self, name, html):
  60         return parse_count(self._search_regex(
  61             rf'([\d,]+)\s+<span[^>]*>{name}</span>', html, name, default=None))
  62
  63     def _real_initialize(self):
  64         homepage = self._download_webpage(
  65             'https://murrtube.net', None, note='Getting session token')
  66         self._request_webpage(
  67             'https://murrtube.net/accept_age_check', None, 'Setting age cookie',
  68             data=urlencode_postdata(self._hidden_inputs(homepage)))
  69
  70     def _real_extract(self, url):
  71         video_id = self._match_id(url)
  72         if video_id.startswith('murrtube:'):
  73             raise ExtractorError('Support for murrtube: prefix URLs is broken')
  74         video_page = self._download_webpage(url, video_id)
  75         video_attrs = extract_attributes(get_element_html_by_id('video', video_page))
  76         playlist = update_url(video_attrs['data-url'], query=None)
  77         video_id = self._search_regex(r'/([\da-f]+)/index.m3u8', playlist, 'video id')
  78
  79         return {
  80             'id': video_id,
  81             'title': remove_end(self._og_search_title(video_page), ' - Murrtube'),
  82             'age_limit': 18,
  83             'formats': self._extract_m3u8_formats(playlist, video_id, 'mp4'),
  84             'description': self._og_search_description(video_page),
  85             'thumbnail': update_url(self._og_search_thumbnail(video_page, default=''), query=None) or None,
  86             'uploader': clean_html(get_element_by_class('pl-1 is-size-6 has-text-lighter', video_page)),
  87             'view_count': self._extract_count('Views', video_page),
  88             'like_count': self._extract_count('Likes', video_page),
  89             'comment_count': self._extract_count('Comments', video_page),
  90         }
  91
  92
  93 class MurrtubeUserIE(InfoExtractor):
  94     _WORKING = False
  95     IE_DESC = 'Murrtube user profile'
  96     _VALID_URL = r'https?://murrtube\.net/(?P<id>[^/]+)$'
  97     _TESTS = [{
  98         'url': 'https://murrtube.net/stormy',
  99         'info_dict': {
 100             'id': 'stormy',
 101         },
 102         'playlist_mincount': 27,
 103     }]
 104     _PAGE_SIZE = 10
 105
 106     def _download_gql(self, video_id, op, note=None, fatal=True):
 107         result = self._download_json(
 108             'https://murrtube.net/graphql',
 109             video_id, note, data=json.dumps(op).encode(), fatal=fatal,
 110             headers={'Content-Type': 'application/json'})
 111         return result['data']
 112
 113     def _fetch_page(self, username, user_id, page):
 114         data = self._download_gql(username, {
 115             'operationName': 'Media',
 116             'variables': {
 117                 'limit': self._PAGE_SIZE,
 118                 'offset': page * self._PAGE_SIZE,
 119                 'sort': 'latest',
 120                 'userId': user_id,
 121             },
 122             'query': '''\
 123 query Media($q: String, $sort: String, $userId: ID, $offset: Int!, $limit: Int!) {
 124   media(q: $q, sort: $sort, userId: $userId, offset: $offset, limit: $limit) {
 125     id
 126     __typename
 127   }
 128 }'''},
 129             f'Downloading page {page + 1}')
 130         if data is None:
 131             raise ExtractorError(f'Failed to retrieve video list for page {page + 1}')
 132
 133         media = data['media']
 134
 135         for entry in media:
 136             yield self.url_result('murrtube:{}'.format(entry['id']), MurrtubeIE.ie_key())
 137
 138     def _real_extract(self, url):
 139         username = self._match_id(url)
 140         data = self._download_gql(username, {
 141             'operationName': 'User',
 142             'variables': {
 143                 'id': username,
 144             },
 145             'query': '''\
 146 query User($id: ID!) {
 147   user(id: $id) {
 148     id
 149     __typename
 150   }
 151 }'''},
 152             'Downloading user info')
 153         if data is None:
 154             raise ExtractorError('Failed to fetch user info')
 155
 156         user = data['user']
 157
 158         entries = OnDemandPagedList(functools.partial(
 159             self._fetch_page, username, user.get('id')), self._PAGE_SIZE)
 160
 161         return self.playlist_result(entries, username)