yt_dlp/extractor/radiko.py

   1 import base64
   2 import random
   3 import re
   4 import urllib.parse
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     ExtractorError,
   9     clean_html,
  10     join_nonempty,
  11     time_seconds,
  12     try_call,
  13     unified_timestamp,
  14     update_url_query,
  15 )
  16 from ..utils.traversal import traverse_obj
  17
  18
  19 class RadikoBaseIE(InfoExtractor):
  20     _GEO_BYPASS = False
  21     _FULL_KEY = None
  22     _HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED = (
  23         'https://c-rpaa.smartstream.ne.jp',
  24         'https://si-c-radiko.smartstream.ne.jp',
  25         'https://tf-f-rpaa-radiko.smartstream.ne.jp',
  26         'https://tf-c-rpaa-radiko.smartstream.ne.jp',
  27         'https://si-f-radiko.smartstream.ne.jp',
  28         'https://rpaa.smartstream.ne.jp',
  29     )
  30     _HOSTS_FOR_TIME_FREE_FFMPEG_SUPPORTED = (
  31         'https://rd-wowza-radiko.radiko-cf.com',
  32         'https://radiko.jp',
  33         'https://f-radiko.smartstream.ne.jp',
  34     )
  35     # Following URL forcibly connects not Time Free but Live
  36     _HOSTS_FOR_LIVE = (
  37         'https://c-radiko.smartstream.ne.jp',
  38     )
  39
  40     def _negotiate_token(self):
  41         _, auth1_handle = self._download_webpage_handle(
  42             'https://radiko.jp/v2/api/auth1', None, 'Downloading authentication page',
  43             headers={
  44                 'x-radiko-app': 'pc_html5',
  45                 'x-radiko-app-version': '0.0.1',
  46                 'x-radiko-device': 'pc',
  47                 'x-radiko-user': 'dummy_user',
  48             })
  49         auth1_header = auth1_handle.headers
  50
  51         auth_token = auth1_header['X-Radiko-AuthToken']
  52         kl = int(auth1_header['X-Radiko-KeyLength'])
  53         ko = int(auth1_header['X-Radiko-KeyOffset'])
  54         raw_partial_key = self._extract_full_key()[ko:ko + kl]
  55         partial_key = base64.b64encode(raw_partial_key).decode()
  56
  57         area_id = self._download_webpage(
  58             'https://radiko.jp/v2/api/auth2', None, 'Authenticating',
  59             headers={
  60                 'x-radiko-device': 'pc',
  61                 'x-radiko-user': 'dummy_user',
  62                 'x-radiko-authtoken': auth_token,
  63                 'x-radiko-partialkey': partial_key,
  64             }).split(',')[0]
  65
  66         if area_id == 'OUT':
  67             self.raise_geo_restricted(countries=['JP'])
  68
  69         auth_data = (auth_token, area_id)
  70         self.cache.store('radiko', 'auth_data', auth_data)
  71         return auth_data
  72
  73     def _auth_client(self):
  74         cachedata = self.cache.load('radiko', 'auth_data')
  75         if cachedata is not None:
  76             response = self._download_webpage(
  77                 'https://radiko.jp/v2/api/auth_check', None, 'Checking cached token', expected_status=401,
  78                 headers={'X-Radiko-AuthToken': cachedata[0], 'X-Radiko-AreaId': cachedata[1]})
  79             if response == 'OK':
  80                 return cachedata
  81         return self._negotiate_token()
  82
  83     def _extract_full_key(self):
  84         if self._FULL_KEY:
  85             return self._FULL_KEY
  86
  87         jscode = self._download_webpage(
  88             'https://radiko.jp/apps/js/playerCommon.js', None,
  89             note='Downloading player js code')
  90         full_key = self._search_regex(
  91             (r"RadikoJSPlayer\([^,]*,\s*(['\"])pc_html5\1,\s*(['\"])(?P<fullkey>[0-9a-f]+)\2,\s*{"),
  92             jscode, 'full key', fatal=False, group='fullkey')
  93
  94         if full_key:
  95             full_key = full_key.encode()
  96         else:  # use only full key ever known
  97             full_key = b'bcd151073c03b352e1ef2fd66c32209da9ca0afa'
  98
  99         self._FULL_KEY = full_key
 100         return full_key
 101
 102     def _find_program(self, video_id, station, cursor):
 103         station_program = self._download_xml(
 104             f'https://radiko.jp/v3/program/station/weekly/{station}.xml', video_id,
 105             note=f'Downloading radio program for {station} station')
 106
 107         prog = None
 108         for p in station_program.findall('.//prog'):
 109             ft_str, to_str = p.attrib['ft'], p.attrib['to']
 110             ft = unified_timestamp(ft_str, False)
 111             to = unified_timestamp(to_str, False)
 112             if ft <= cursor and cursor < to:
 113                 prog = p
 114                 break
 115         if not prog:
 116             raise ExtractorError('Cannot identify radio program to download!')
 117         assert ft, to
 118         return prog, station_program, ft, ft_str, to_str
 119
 120     def _extract_formats(self, video_id, station, is_onair, ft, cursor, auth_token, area_id, query):
 121         m3u8_playlist_data = self._download_xml(
 122             f'https://radiko.jp/v3/station/stream/pc_html5/{station}.xml', video_id,
 123             note='Downloading stream information')
 124
 125         formats = []
 126         found = set()
 127
 128         timefree_int = 0 if is_onair else 1
 129
 130         for element in m3u8_playlist_data.findall(f'.//url[@timefree="{timefree_int}"]/playlist_create_url'):
 131             pcu = element.text
 132             if pcu in found:
 133                 continue
 134             found.add(pcu)
 135             playlist_url = update_url_query(pcu, {
 136                 'station_id': station,
 137                 **query,
 138                 'l': '15',
 139                 'lsid': ''.join(random.choices('0123456789abcdef', k=32)),
 140                 'type': 'b',
 141             })
 142
 143             time_to_skip = None if is_onair else cursor - ft
 144
 145             domain = urllib.parse.urlparse(playlist_url).netloc
 146             subformats = self._extract_m3u8_formats(
 147                 playlist_url, video_id, ext='m4a',
 148                 live=True, fatal=False, m3u8_id=domain,
 149                 note=f'Downloading m3u8 information from {domain}',
 150                 headers={
 151                     'X-Radiko-AreaId': area_id,
 152                     'X-Radiko-AuthToken': auth_token,
 153                 })
 154             for sf in subformats:
 155                 if (is_onair ^ pcu.startswith(self._HOSTS_FOR_LIVE)) or (
 156                         not is_onair and pcu.startswith(self._HOSTS_FOR_TIME_FREE_FFMPEG_UNSUPPORTED)):
 157                     sf['preference'] = -100
 158                     sf['format_note'] = 'not preferred'
 159                 if not is_onair and timefree_int == 1 and time_to_skip:
 160                     sf['downloader_options'] = {'ffmpeg_args': ['-ss', str(time_to_skip)]}
 161             formats.extend(subformats)
 162
 163         return formats
 164
 165     def _extract_performers(self, prog):
 166         return traverse_obj(prog, (
 167             'pfm/text()', ..., {lambda x: re.split(r'[/／、　,，]', x)}, ..., {str.strip})) or None
 168
 169
 170 class RadikoIE(RadikoBaseIE):
 171     _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<timestring>\d+)'
 172
 173     _TESTS = [{
 174         # QRR (文化放送) station provides <desc>
 175         'url': 'https://radiko.jp/#!/ts/QRR/20210425101300',
 176         'only_matching': True,
 177     }, {
 178         # FMT (TOKYO FM) station does not provide <desc>
 179         'url': 'https://radiko.jp/#!/ts/FMT/20210810150000',
 180         'only_matching': True,
 181     }, {
 182         'url': 'https://radiko.jp/#!/ts/JOAK-FM/20210509090000',
 183         'only_matching': True,
 184     }]
 185
 186     def _real_extract(self, url):
 187         station, timestring = self._match_valid_url(url).group('station', 'timestring')
 188         video_id = join_nonempty(station, timestring)
 189         vid_int = unified_timestamp(timestring, False)
 190         prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
 191
 192         auth_token, area_id = self._auth_client()
 193
 194         return {
 195             'id': video_id,
 196             'title': try_call(lambda: prog.find('title').text),
 197             'cast': self._extract_performers(prog),
 198             'description': clean_html(try_call(lambda: prog.find('info').text)),
 199             'uploader': try_call(lambda: station_program.find('.//name').text),
 200             'uploader_id': station,
 201             'timestamp': vid_int,
 202             'duration': try_call(lambda: unified_timestamp(radio_end, False) - unified_timestamp(radio_begin, False)),
 203             'is_live': True,
 204             'formats': self._extract_formats(
 205                 video_id=video_id, station=station, is_onair=False,
 206                 ft=ft, cursor=vid_int, auth_token=auth_token, area_id=area_id,
 207                 query={
 208                     'start_at': radio_begin,
 209                     'ft': radio_begin,
 210                     'end_at': radio_end,
 211                     'to': radio_end,
 212                     'seek': timestring,
 213                 },
 214             ),
 215         }
 216
 217
 218 class RadikoRadioIE(RadikoBaseIE):
 219     _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/live/(?P<id>[A-Z0-9-]+)'
 220
 221     _TESTS = [{
 222         # QRR (文化放送) station provides <desc>
 223         'url': 'https://radiko.jp/#!/live/QRR',
 224         'only_matching': True,
 225     }, {
 226         # FMT (TOKYO FM) station does not provide <desc>
 227         'url': 'https://radiko.jp/#!/live/FMT',
 228         'only_matching': True,
 229     }, {
 230         'url': 'https://radiko.jp/#!/live/JOAK-FM',
 231         'only_matching': True,
 232     }]
 233
 234     def _real_extract(self, url):
 235         station = self._match_id(url)
 236         self.report_warning('Downloader will not stop at the end of the program! Press Ctrl+C to stop')
 237
 238         auth_token, area_id = self._auth_client()
 239         # get current time in JST (GMT+9:00 w/o DST)
 240         vid_now = time_seconds(hours=9)
 241
 242         prog, station_program, ft, _, _ = self._find_program(station, station, vid_now)
 243
 244         title = prog.find('title').text
 245         description = clean_html(prog.find('info').text)
 246         station_name = station_program.find('.//name').text
 247
 248         formats = self._extract_formats(
 249             video_id=station, station=station, is_onair=True,
 250             ft=ft, cursor=vid_now, auth_token=auth_token, area_id=area_id,
 251             query={})
 252
 253         return {
 254             'id': station,
 255             'title': title,
 256             'cast': self._extract_performers(prog),
 257             'description': description,
 258             'uploader': station_name,
 259             'uploader_id': station,
 260             'timestamp': ft,
 261             'formats': formats,
 262             'is_live': True,
 263         }