[fd/NiconicoDmc] Remove downloader (#12916)

Authored by: doe1080

README.md

@@ -1799,9 +1799,6 @@ The following extractors use this feature:
 #### vikichannel
 * `video_types`: Types of videos to download - one or more of `episodes`, `movies`, `clips`, `trailers`
 
-#### niconico
-* `segment_duration`: Segment duration in milliseconds for HLS-DMC formats. Use it at your own risk since this feature **may result in your account termination.**
-
 #### youtubewebarchive
 * `check_all`: Try to check more at the cost of more requests. One or more of `thumbnails`, `captures`
 

yt_dlp/downloader/__init__.py

@@ -30,7 +30,7 @@ from .hls import HlsFD
 from .http import HttpFD
 from .ism import IsmFD
 from .mhtml import MhtmlFD
-from .niconico import NiconicoDmcFD, NiconicoLiveFD
+from .niconico import NiconicoLiveFD
 from .rtmp import RtmpFD
 from .rtsp import RtspFD
 from .websocket import WebSocketFragmentFD
@@ -50,7 +50,6 @@ PROTOCOL_MAP = {
     'http_dash_segments_generator': DashSegmentsFD,
     'ism': IsmFD,
     'mhtml': MhtmlFD,
-    'niconico_dmc': NiconicoDmcFD,
     'niconico_live': NiconicoLiveFD,
     'fc2_live': FC2LiveFD,
     'websocket_frag': WebSocketFragmentFD,
@@ -67,7 +66,6 @@ def shorten_protocol_name(proto, simplify=False):
         'rtmp_ffmpeg': 'rtmpF',
         'http_dash_segments': 'dash',
         'http_dash_segments_generator': 'dashG',
-        'niconico_dmc': 'dmc',
         'websocket_frag': 'WSfrag',
     }
     if simplify:

yt_dlp/downloader/niconico.py

@@ -2,60 +2,12 @@ import json
 import threading
 import time
 
-from . import get_suitable_downloader
 from .common import FileDownloader
 from .external import FFmpegFD
 from ..networking import Request
 from ..utils import DownloadError, str_or_none, try_get
 
 
-class NiconicoDmcFD(FileDownloader):
-    """ Downloading niconico douga from DMC with heartbeat """
-
-    def real_download(self, filename, info_dict):
-        from ..extractor.niconico import NiconicoIE
-
-        self.to_screen(f'[{self.FD_NAME}] Downloading from DMC')
-        ie = NiconicoIE(self.ydl)
-        info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
-
-        fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
-
-        success = download_complete = False
-        timer = [None]
-        heartbeat_lock = threading.Lock()
-        heartbeat_url = heartbeat_info_dict['url']
-        heartbeat_data = heartbeat_info_dict['data'].encode()
-        heartbeat_interval = heartbeat_info_dict.get('interval', 30)
-
-        request = Request(heartbeat_url, heartbeat_data)
-
-        def heartbeat():
-            try:
-                self.ydl.urlopen(request).read()
-            except Exception:
-                self.to_screen(f'[{self.FD_NAME}] Heartbeat failed')
-
-            with heartbeat_lock:
-                if not download_complete:
-                    timer[0] = threading.Timer(heartbeat_interval, heartbeat)
-                    timer[0].start()
-
-        heartbeat_info_dict['ping']()
-        self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval))
-        try:
-            heartbeat()
-            if type(fd).__name__ == 'HlsFD':
-                info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0])
-            success = fd.real_download(filename, info_dict)
-        finally:
-            if heartbeat_lock:
-                with heartbeat_lock:
-                    timer[0].cancel()
-                    download_complete = True
-        return success
-
-
 class NiconicoLiveFD(FileDownloader):
     """ Downloads niconico live without being stopped """
 

yt_dlp/extractor/dacast.py

@@ -9,6 +9,7 @@ from ..utils import (
     ExtractorError,
     classproperty,
     float_or_none,
+    parse_qs,
     traverse_obj,
     url_or_none,
 )
@@ -91,11 +92,15 @@ class DacastVODIE(DacastBaseIE):
         # Rotates every so often, but hardcode a fallback in case of JS change/breakage before rotation
         return self._search_regex(
             r'\bUSP_SIGNING_SECRET\s*=\s*(["\'])(?P<secret>(?:(?!\1).)+)', player_js,
-            'usp signing secret', group='secret', fatal=False) or 'odnInCGqhvtyRTtIiddxtuRtawYYICZP'
+            'usp signing secret', group='secret', fatal=False) or 'hGDtqMKYVeFdofrAfFmBcrsakaZELajI'
 
     def _real_extract(self, url):
         user_id, video_id = self._match_valid_url(url).group('user_id', 'id')
-        query = {'contentId': f'{user_id}-vod-{video_id}', 'provider': 'universe'}
+        query = {
+            'contentId': f'{user_id}-vod-{video_id}',
+            'provider': 'universe',
+            **traverse_obj(url, ({parse_qs}, 'uss_token', {'signedKey': -1})),
+        }
         info = self._download_json(self._API_INFO_URL, video_id, query=query, fatal=False)
         access = self._download_json(
             'https://playback.dacast.com/content/access', video_id,

yt_dlp/extractor/mlb.py

@@ -365,13 +365,15 @@ mutation initPlaybackSession(
                 'All videos are only available to registered users', method='password')
 
     def _set_device_id(self, username):
-        if not self._device_id:
+        if self._device_id:
-            self._device_id = self.cache.load(
+            return
-                self._NETRC_MACHINE, 'device_ids', default={}).get(username)
+        device_id_cache = self.cache.load(self._NETRC_MACHINE, 'device_ids', default={})
+        self._device_id = device_id_cache.get(username)
         if self._device_id:
             return
         self._device_id = str(uuid.uuid4())
-        self.cache.store(self._NETRC_MACHINE, 'device_ids', {username: self._device_id})
+        device_id_cache[username] = self._device_id
+        self.cache.store(self._NETRC_MACHINE, 'device_ids', device_id_cache)
 
     def _perform_login(self, username, password):
         try:

yt_dlp/extractor/niconico.py

@@ -16,7 +16,6 @@ from ..utils import (
     determine_ext,
     float_or_none,
     int_or_none,
-    join_nonempty,
     parse_duration,
     parse_iso8601,
     parse_qs,
@@ -181,13 +180,6 @@ class NiconicoIE(InfoExtractor):
 
     _VALID_URL = r'https?://(?:(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch|nico\.ms)/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
-    _API_HEADERS = {
-        'X-Frontend-ID': '6',
-        'X-Frontend-Version': '0',
-        'X-Niconico-Language': 'en-us',
-        'Referer': 'https://www.nicovideo.jp/',
-        'Origin': 'https://www.nicovideo.jp',
-    }
 
     def _perform_login(self, username, password):
         login_ok = True
@@ -229,181 +221,6 @@ class NiconicoIE(InfoExtractor):
             self.report_warning('Unable to log in: bad username or password')
         return login_ok
 
-    def _get_heartbeat_info(self, info_dict):
-        video_id, video_src_id, audio_src_id = info_dict['url'].split(':')[1].split('/')
-        dmc_protocol = info_dict['expected_protocol']
-
-        api_data = (
-            info_dict.get('_api_data')
-            or self._parse_json(
-                self._html_search_regex(
-                    'data-api-data="([^"]+)"',
-                    self._download_webpage('https://www.nicovideo.jp/watch/' + video_id, video_id),
-                    'API data', default='{}'),
-                video_id))
-
-        session_api_data = try_get(api_data, lambda x: x['media']['delivery']['movie']['session'])
-        session_api_endpoint = try_get(session_api_data, lambda x: x['urls'][0])
-
-        def ping():
-            tracking_id = traverse_obj(api_data, ('media', 'delivery', 'trackingId'))
-            if tracking_id:
-                tracking_url = update_url_query('https://nvapi.nicovideo.jp/v1/2ab0cbaa/watch', {'t': tracking_id})
-                watch_request_response = self._download_json(
-                    tracking_url, video_id,
-                    note='Acquiring permission for downloading video', fatal=False,
-                    headers=self._API_HEADERS)
-                if traverse_obj(watch_request_response, ('meta', 'status')) != 200:
-                    self.report_warning('Failed to acquire permission for playing video. Video download may fail.')
-
-        yesno = lambda x: 'yes' if x else 'no'
-
-        if dmc_protocol == 'http':
-            protocol = 'http'
-            protocol_parameters = {
-                'http_output_download_parameters': {
-                    'use_ssl': yesno(session_api_data['urls'][0]['isSsl']),
-                    'use_well_known_port': yesno(session_api_data['urls'][0]['isWellKnownPort']),
-                },
-            }
-        elif dmc_protocol == 'hls':
-            protocol = 'm3u8'
-            segment_duration = try_get(self._configuration_arg('segment_duration'), lambda x: int(x[0])) or 6000
-            parsed_token = self._parse_json(session_api_data['token'], video_id)
-            encryption = traverse_obj(api_data, ('media', 'delivery', 'encryption'))
-            protocol_parameters = {
-                'hls_parameters': {
-                    'segment_duration': segment_duration,
-                    'transfer_preset': '',
-                    'use_ssl': yesno(session_api_data['urls'][0]['isSsl']),
-                    'use_well_known_port': yesno(session_api_data['urls'][0]['isWellKnownPort']),
-                },
-            }
-            if 'hls_encryption' in parsed_token and encryption:
-                protocol_parameters['hls_parameters']['encryption'] = {
-                    parsed_token['hls_encryption']: {
-                        'encrypted_key': encryption['encryptedKey'],
-                        'key_uri': encryption['keyUri'],
-                    },
-                }
-            else:
-                protocol = 'm3u8_native'
-        else:
-            raise ExtractorError(f'Unsupported DMC protocol: {dmc_protocol}')
-
-        session_response = self._download_json(
-            session_api_endpoint['url'], video_id,
-            query={'_format': 'json'},
-            headers={'Content-Type': 'application/json'},
-            note='Downloading JSON metadata for {}'.format(info_dict['format_id']),
-            data=json.dumps({
-                'session': {
-                    'client_info': {
-                        'player_id': session_api_data.get('playerId'),
-                    },
-                    'content_auth': {
-                        'auth_type': try_get(session_api_data, lambda x: x['authTypes'][session_api_data['protocols'][0]]),
-                        'content_key_timeout': session_api_data.get('contentKeyTimeout'),
-                        'service_id': 'nicovideo',
-                        'service_user_id': session_api_data.get('serviceUserId'),
-                    },
-                    'content_id': session_api_data.get('contentId'),
-                    'content_src_id_sets': [{
-                        'content_src_ids': [{
-                            'src_id_to_mux': {
-                                'audio_src_ids': [audio_src_id],
-                                'video_src_ids': [video_src_id],
-                            },
-                        }],
-                    }],
-                    'content_type': 'movie',
-                    'content_uri': '',
-                    'keep_method': {
-                        'heartbeat': {
-                            'lifetime': session_api_data.get('heartbeatLifetime'),
-                        },
-                    },
-                    'priority': session_api_data['priority'],
-                    'protocol': {
-                        'name': 'http',
-                        'parameters': {
-                            'http_parameters': {
-                                'parameters': protocol_parameters,
-                            },
-                        },
-                    },
-                    'recipe_id': session_api_data.get('recipeId'),
-                    'session_operation_auth': {
-                        'session_operation_auth_by_signature': {
-                            'signature': session_api_data.get('signature'),
-                            'token': session_api_data.get('token'),
-                        },
-                    },
-                    'timing_constraint': 'unlimited',
-                },
-            }).encode())
-
-        info_dict['url'] = session_response['data']['session']['content_uri']
-        info_dict['protocol'] = protocol
-
-        # get heartbeat info
-        heartbeat_info_dict = {
-            'url': session_api_endpoint['url'] + '/' + session_response['data']['session']['id'] + '?_format=json&_method=PUT',
-            'data': json.dumps(session_response['data']),
-            # interval, convert milliseconds to seconds, then halve to make a buffer.
-            'interval': float_or_none(session_api_data.get('heartbeatLifetime'), scale=3000),
-            'ping': ping,
-        }
-
-        return info_dict, heartbeat_info_dict
-
-    def _extract_format_for_quality(self, video_id, audio_quality, video_quality, dmc_protocol):
-
-        if not audio_quality.get('isAvailable') or not video_quality.get('isAvailable'):
-            return None
-
-        format_id = '-'.join(
-            [remove_start(s['id'], 'archive_') for s in (video_quality, audio_quality)] + [dmc_protocol])
-
-        vid_qual_label = traverse_obj(video_quality, ('metadata', 'label'))
-
-        return {
-            'url': 'niconico_dmc:{}/{}/{}'.format(video_id, video_quality['id'], audio_quality['id']),
-            'format_id': format_id,
-            'format_note': join_nonempty('DMC', vid_qual_label, dmc_protocol.upper(), delim=' '),
-            'ext': 'mp4',  # Session API are used in HTML5, which always serves mp4
-            'acodec': 'aac',
-            'vcodec': 'h264',
-            **traverse_obj(audio_quality, ('metadata', {
-                'abr': ('bitrate', {float_or_none(scale=1000)}),
-                'asr': ('samplingRate', {int_or_none}),
-            })),
-            **traverse_obj(video_quality, ('metadata', {
-                'vbr': ('bitrate', {float_or_none(scale=1000)}),
-                'height': ('resolution', 'height', {int_or_none}),
-                'width': ('resolution', 'width', {int_or_none}),
-            })),
-            'quality': -2 if 'low' in video_quality['id'] else None,
-            'protocol': 'niconico_dmc',
-            'expected_protocol': dmc_protocol,  # XXX: This is not a documented field
-            'http_headers': {
-                'Origin': 'https://www.nicovideo.jp',
-                'Referer': 'https://www.nicovideo.jp/watch/' + video_id,
-            },
-        }
-
-    def _yield_dmc_formats(self, api_data, video_id):
-        dmc_data = traverse_obj(api_data, ('media', 'delivery', 'movie'))
-        audios = traverse_obj(dmc_data, ('audios', ..., {dict}))
-        videos = traverse_obj(dmc_data, ('videos', ..., {dict}))
-        protocols = traverse_obj(dmc_data, ('session', 'protocols', ..., {str}))
-        if not all((audios, videos, protocols)):
-            return
-
-        for audio_quality, video_quality, protocol in itertools.product(audios, videos, protocols):
-            if fmt := self._extract_format_for_quality(video_id, audio_quality, video_quality, protocol):
-                yield fmt
-
     def _yield_dms_formats(self, api_data, video_id):
         fmt_filter = lambda _, v: v['isAvailable'] and v['id']
         videos = traverse_obj(api_data, ('media', 'domand', 'videos', fmt_filter))
@@ -485,8 +302,8 @@ class NiconicoIE(InfoExtractor):
             'needs_premium': ('isPremium', {bool}),
             'needs_subscription': ('isAdmission', {bool}),
         })) or {'needs_auth': True}))
-        formats = [*self._yield_dmc_formats(api_data, video_id),
+
-                   *self._yield_dms_formats(api_data, video_id)]
+        formats = list(self._yield_dms_formats(api_data, video_id))
         if not formats:
             fail_msg = clean_html(self._html_search_regex(
                 r'<p[^>]+\bclass="fail-message"[^>]*>(?P<msg>.+?)</p>',