yt_dlp/extractor/manyvids.py

   1 import re
   2
   3 from .common import InfoExtractor
   4 from ..utils import (
   5     determine_ext,
   6     extract_attributes,
   7     int_or_none,
   8     str_to_int,
   9     url_or_none,
  10     urlencode_postdata,
  11 )
  12
  13
  14 class ManyVidsIE(InfoExtractor):
  15     _WORKING = False
  16     _VALID_URL = r'(?i)https?://(?:www\.)?manyvids\.com/video/(?P<id>\d+)'
  17     _TESTS = [{
  18         # preview video
  19         'url': 'https://www.manyvids.com/Video/133957/everthing-about-me/',
  20         'md5': '03f11bb21c52dd12a05be21a5c7dcc97',
  21         'info_dict': {
  22             'id': '133957',
  23             'ext': 'mp4',
  24             'title': 'everthing about me (Preview)',
  25             'uploader': 'ellyxxix',
  26             'view_count': int,
  27             'like_count': int,
  28         },
  29     }, {
  30         # full video
  31         'url': 'https://www.manyvids.com/Video/935718/MY-FACE-REVEAL/',
  32         'md5': 'bb47bab0e0802c2a60c24ef079dfe60f',
  33         'info_dict': {
  34             'id': '935718',
  35             'ext': 'mp4',
  36             'title': 'MY FACE REVEAL',
  37             'description': 'md5:ec5901d41808b3746fed90face161612',
  38             'uploader': 'Sarah Calanthe',
  39             'view_count': int,
  40             'like_count': int,
  41         },
  42     }]
  43
  44     def _real_extract(self, url):
  45         video_id = self._match_id(url)
  46
  47         real_url = 'https://www.manyvids.com/video/%s/gtm.js' % (video_id, )
  48         try:
  49             webpage = self._download_webpage(real_url, video_id)
  50         except Exception:
  51             # probably useless fallback
  52             webpage = self._download_webpage(url, video_id)
  53
  54         info = self._search_regex(
  55             r'''(<div\b[^>]*\bid\s*=\s*(['"])pageMetaDetails\2[^>]*>)''',
  56             webpage, 'meta details', default='')
  57         info = extract_attributes(info)
  58
  59         player = self._search_regex(
  60             r'''(<div\b[^>]*\bid\s*=\s*(['"])rmpPlayerStream\2[^>]*>)''',
  61             webpage, 'player details', default='')
  62         player = extract_attributes(player)
  63
  64         video_urls_and_ids = (
  65             (info.get('data-meta-video'), 'video'),
  66             (player.get('data-video-transcoded'), 'transcoded'),
  67             (player.get('data-video-filepath'), 'filepath'),
  68             (self._og_search_video_url(webpage, secure=False, default=None), 'og_video'),
  69         )
  70
  71         def txt_or_none(s, default=None):
  72             return (s.strip() or default) if isinstance(s, str) else default
  73
  74         uploader = txt_or_none(info.get('data-meta-author'))
  75
  76         def mung_title(s):
  77             if uploader:
  78                 s = re.sub(r'^\s*%s\s+[|-]' % (re.escape(uploader), ), '', s)
  79             return txt_or_none(s)
  80
  81         title = (
  82             mung_title(info.get('data-meta-title'))
  83             or self._html_search_regex(
  84                 (r'<span[^>]+class=["\']item-title[^>]+>([^<]+)',
  85                  r'<h2[^>]+class=["\']h2 m-0["\'][^>]*>([^<]+)'),
  86                 webpage, 'title', default=None)
  87             or self._html_search_meta(
  88                 'twitter:title', webpage, 'title', fatal=True))
  89
  90         title = re.sub(r'\s*[|-]\s+ManyVids\s*$', '', title) or title
  91
  92         if any(p in webpage for p in ('preview_videos', '_preview.mp4')):
  93             title += ' (Preview)'
  94
  95         mv_token = self._search_regex(
  96             r'data-mvtoken=(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
  97             'mv token', default=None, group='value')
  98
  99         if mv_token:
 100             # Sets some cookies
 101             self._download_webpage(
 102                 'https://www.manyvids.com/includes/ajax_repository/you_had_me_at_hello.php',
 103                 video_id, note='Setting format cookies', fatal=False,
 104                 data=urlencode_postdata({
 105                     'mvtoken': mv_token,
 106                     'vid': video_id,
 107                 }), headers={
 108                     'Referer': url,
 109                     'X-Requested-With': 'XMLHttpRequest'
 110                 })
 111
 112         formats = []
 113         for v_url, fmt in video_urls_and_ids:
 114             v_url = url_or_none(v_url)
 115             if not v_url:
 116                 continue
 117             if determine_ext(v_url) == 'm3u8':
 118                 formats.extend(self._extract_m3u8_formats(
 119                     v_url, video_id, 'mp4', entry_protocol='m3u8_native',
 120                     m3u8_id='hls'))
 121             else:
 122                 formats.append({
 123                     'url': v_url,
 124                     'format_id': fmt,
 125                 })
 126
 127         self._remove_duplicate_formats(formats)
 128
 129         for f in formats:
 130             if f.get('height') is None:
 131                 f['height'] = int_or_none(
 132                     self._search_regex(r'_(\d{2,3}[02468])_', f['url'], 'video height', default=None))
 133             if '/preview/' in f['url']:
 134                 f['format_id'] = '_'.join(filter(None, (f.get('format_id'), 'preview')))
 135                 f['preference'] = -10
 136             if 'transcoded' in f['format_id']:
 137                 f['preference'] = f.get('preference', -1) - 1
 138
 139         def get_likes():
 140             likes = self._search_regex(
 141                 r'''(<a\b[^>]*\bdata-id\s*=\s*(['"])%s\2[^>]*>)''' % (video_id, ),
 142                 webpage, 'likes', default='')
 143             likes = extract_attributes(likes)
 144             return int_or_none(likes.get('data-likes'))
 145
 146         def get_views():
 147             return str_to_int(self._html_search_regex(
 148                 r'''(?s)<span\b[^>]*\bclass\s*=["']views-wrapper\b[^>]+>.+?<span\b[^>]+>\s*(\d[\d,.]*)\s*</span>''',
 149                 webpage, 'view count', default=None))
 150
 151         return {
 152             'id': video_id,
 153             'title': title,
 154             'formats': formats,
 155             'description': txt_or_none(info.get('data-meta-description')),
 156             'uploader': txt_or_none(info.get('data-meta-author')),
 157             'thumbnail': (
 158                 url_or_none(info.get('data-meta-image'))
 159                 or url_or_none(player.get('data-video-screenshot'))),
 160             'view_count': get_views(),
 161             'like_count': get_likes(),
 162         }