[ie/niconico] Fix login support (#13008)

Authored by: doe1080

yt_dlp/extractor/bpb.py

@@ -7,6 +7,7 @@ from ..utils import (
     join_nonempty,
     js_to_json,
     mimetype2ext,
+    parse_resolution,
     unified_strdate,
     url_or_none,
     urljoin,
@@ -110,24 +111,23 @@ class BpbIE(InfoExtractor):
 
         return attributes
 
-    @staticmethod
-    def _process_source(source):
+    def _process_source(self, source):
         url = url_or_none(source['src'])
         if not url:
             return None
 
         source_type = source.get('type', '')
         extension = mimetype2ext(source_type)
-        is_video = source_type.startswith('video')
-        note = url.rpartition('.')[0].rpartition('_')[2] if is_video else None
+        note = self._search_regex(r'[_-]([a-z]+)\.[\da-z]+(?:$|\?)', url, 'note', default=None)
 
         return {
             'url': url,
             'ext': extension,
-            'vcodec': None if is_video else 'none',
+            'vcodec': None if source_type.startswith('video') else 'none',
             'quality': 10 if note == 'high' else 0,
             'format_note': note,
             'format_id': join_nonempty(extension, note),
+            **parse_resolution(source.get('label')),
         }
 
     def _real_extract(self, url):

yt_dlp/extractor/niconico.py

@@ -23,7 +23,6 @@ from ..utils import (
     qualities,
     remove_start,
     str_or_none,
-    traverse_obj,
     try_get,
     unescapeHTML,
     unified_timestamp,
@@ -33,13 +32,70 @@ from ..utils import (
     urlencode_postdata,
     urljoin,
 )
+from ..utils.traversal import find_element, traverse_obj
 
 
-class NiconicoIE(InfoExtractor):
+class NiconicoBaseIE(InfoExtractor):
+    _GEO_BYPASS = False
+    _GEO_COUNTRIES = ['JP']
+    _LOGIN_BASE = 'https://account.nicovideo.jp'
+    _NETRC_MACHINE = 'niconico'
+
+    @property
+    def is_logged_in(self):
+        return bool(self._get_cookies('https://www.nicovideo.jp').get('user_session'))
+
+    def _raise_login_error(self, message, expected=True):
+        raise ExtractorError(f'Unable to login: {message}', expected=expected)
+
+    def _perform_login(self, username, password):
+        if self.is_logged_in:
+            return
+
+        self._request_webpage(
+            f'{self._LOGIN_BASE}/login', None, 'Requesting session cookies')
+        webpage = self._download_webpage(
+            f'{self._LOGIN_BASE}/login/redirector', None,
+            'Logging in', 'Unable to log in', headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': f'{self._LOGIN_BASE}/login',
+            }, data=urlencode_postdata({
+                'mail_tel': username,
+                'password': password,
+            }))
+
+        if self.is_logged_in:
+            return
+        elif err_msg := traverse_obj(webpage, (
+            {find_element(cls='notice error')}, {find_element(cls='notice__text')}, {clean_html},
+        )):
+            self._raise_login_error(err_msg or 'Invalid username or password')
+        elif 'oneTimePw' in webpage:
+            post_url = self._search_regex(
+                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage, 'post url', group='url')
+            mfa, urlh = self._download_webpage_handle(
+                urljoin(self._LOGIN_BASE, post_url), None,
+                'Performing MFA', 'Unable to complete MFA', headers={
+                    'Content-Type': 'application/x-www-form-urlencoded',
+                }, data=urlencode_postdata({
+                    'otp': self._get_tfa_info('6 digit number shown on app'),
+                }))
+            if self.is_logged_in:
+                return
+            elif 'error-code' in parse_qs(urlh.url):
+                err_msg = traverse_obj(mfa, ({find_element(cls='pageMainMsg')}, {clean_html}))
+                self._raise_login_error(err_msg or 'MFA session expired')
+            elif 'formError' in mfa:
+                err_msg = traverse_obj(mfa, (
+                    {find_element(cls='formError')}, {find_element(tag='div')}, {clean_html}))
+                self._raise_login_error(err_msg or 'MFA challenge failed')
+
+        self._raise_login_error('Unexpected login error', expected=False)
+
+
+class NiconicoIE(NiconicoBaseIE):
     IE_NAME = 'niconico'
     IE_DESC = 'ニコニコ動画'
-    _GEO_COUNTRIES = ['JP']
-    _GEO_BYPASS = False
 
     _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
@@ -179,47 +235,6 @@ class NiconicoIE(InfoExtractor):
     }]
 
     _VALID_URL = r'https?://(?:(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch|nico\.ms)/(?P<id>(?:[a-z]{2})?[0-9]+)'
-    _NETRC_MACHINE = 'niconico'
-
-    def _perform_login(self, username, password):
-        login_ok = True
-        login_form_strs = {
-            'mail_tel': username,
-            'password': password,
-        }
-        self._request_webpage(
-            'https://account.nicovideo.jp/login', None,
-            note='Acquiring Login session')
-        page = self._download_webpage(
-            'https://account.nicovideo.jp/login/redirector?show_button_twitter=1&site=niconico&show_button_facebook=1', None,
-            note='Logging in', errnote='Unable to log in',
-            data=urlencode_postdata(login_form_strs),
-            headers={
-                'Referer': 'https://account.nicovideo.jp/login',
-                'Content-Type': 'application/x-www-form-urlencoded',
-            })
-        if 'oneTimePw' in page:
-            post_url = self._search_regex(
-                r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page, 'post url', group='url')
-            page = self._download_webpage(
-                urljoin('https://account.nicovideo.jp', post_url), None,
-                note='Performing MFA', errnote='Unable to complete MFA',
-                data=urlencode_postdata({
-                    'otp': self._get_tfa_info('6 digits code'),
-                }), headers={
-                    'Content-Type': 'application/x-www-form-urlencoded',
-                })
-            if 'oneTimePw' in page or 'formError' in page:
-                err_msg = self._html_search_regex(
-                    r'formError["\']+>(.*?)</div>', page, 'form_error',
-                    default='There\'s an error but the message can\'t be parsed.',
-                    flags=re.DOTALL)
-                self.report_warning(f'Unable to log in: MFA challenge failed, "{err_msg}"')
-                return False
-        login_ok = 'class="notice error"' not in page
-        if not login_ok:
-            self.report_warning('Unable to log in: bad username or password')
-        return login_ok
 
     def _yield_dms_formats(self, api_data, video_id):
         fmt_filter = lambda _, v: v['isAvailable'] and v['id']
@@ -738,7 +753,7 @@ class NiconicoUserIE(InfoExtractor):
         return self.playlist_result(self._entries(list_id), list_id)
 
 
-class NiconicoLiveIE(InfoExtractor):
+class NiconicoLiveIE(NiconicoBaseIE):
     IE_NAME = 'niconico:live'
     IE_DESC = 'ニコニコ生放送'
     _VALID_URL = r'https?://(?:sp\.)?live2?\.nicovideo\.jp/(?:watch|gate)/(?P<id>lv\d+)'

yt_dlp/extractor/reddit.py

@@ -388,7 +388,8 @@ class RedditIE(InfoExtractor):
                     })
             if entries:
                 return self.playlist_result(entries, video_id, **info)
-            raise ExtractorError('No media found', expected=True)
+            self.raise_no_formats('No media found', expected=True, video_id=video_id)
+            return {**info, 'id': video_id}
 
         # Check if media is hosted on reddit:
         reddit_video = traverse_obj(data, (

yt_dlp/extractor/tv2dk.py

@@ -2,12 +2,13 @@ import json
 import re
 
 from .common import InfoExtractor
+from .jwplatform import JWPlatformIE
 from ..utils import (
     determine_ext,
-    extract_attributes,
     js_to_json,
     url_or_none,
 )
+from ..utils.traversal import find_element, traverse_obj
 
 
 class TV2DKIE(InfoExtractor):
@@ -21,35 +22,46 @@ class TV2DKIE(InfoExtractor):
                             tv2fyn|
                             tv2east|
                             tv2lorry|
-                            tv2nord
+                            tv2nord|
+                            tv2kosmopol
                         )\.dk/
-                        (:[^/]+/)*
+                        (?:[^/?#]+/)*
                         (?P<id>[^/?\#&]+)
                     '''
     _TESTS = [{
         'url': 'https://www.tvsyd.dk/nyheder/28-10-2019/1930/1930-28-okt-2019?autoplay=1#player',
         'info_dict': {
-            'id': '0_52jmwa0p',
+            'id': 'sPp5z21q',
             'ext': 'mp4',
             'title': '19:30 - 28. okt. 2019',
-            'timestamp': 1572290248,
+            'description': '',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/sPp5z21q/poster.jpg?width=720',
+            'timestamp': 1572287400,
             'upload_date': '20191028',
-            'uploader_id': 'tvsyd',
-            'duration': 1347,
-            'view_count': int,
         },
-        'add_ie': ['Kaltura'],
     }, {
         'url': 'https://www.tv2lorry.dk/gadekamp/gadekamp-6-hoejhuse-i-koebenhavn',
         'info_dict': {
-            'id': '1_7iwll9n0',
+            'id': 'oD9cyq0m',
             'ext': 'mp4',
-            'upload_date': '20211027',
             'title': 'Gadekamp #6 - Højhuse i København',
-            'uploader_id': 'tv2lorry',
-            'timestamp': 1635345229,
+            'description': '',
+            'thumbnail': 'https://cdn.jwplayer.com/v2/media/oD9cyq0m/poster.jpg?width=720',
+            'timestamp': 1635348600,
+            'upload_date': '20211027',
         },
-        'add_ie': ['Kaltura'],
+    }, {
+        'url': 'https://www.tvsyd.dk/haderslev/x-factor-brodre-fulde-af-selvtillid-er-igen-hjemme-hos-mor-vores-diagnoser-har-vaeret-en-fordel',
+        'info_dict': {
+            'id': 'x-factor-brodre-fulde-af-selvtillid-er-igen-hjemme-hos-mor-vores-diagnoser-har-vaeret-en-fordel',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://www.tv2ostjylland.dk/aarhus/dom-kan-fa-alvorlige-konsekvenser',
+        'info_dict': {
+            'id': 'dom-kan-fa-alvorlige-konsekvenser',
+        },
+        'playlist_count': 3,
     }, {
         'url': 'https://www.tv2ostjylland.dk/artikel/minister-gaar-ind-i-sag-om-diabetes-teknologi',
         'only_matching': True,
@@ -71,40 +83,22 @@ class TV2DKIE(InfoExtractor):
     }, {
         'url': 'https://www.tv2nord.dk/artikel/dybt-uacceptabelt',
         'only_matching': True,
+    }, {
+        'url': 'https://www.tv2kosmopol.dk/metropolen/chaufforer-beordres-til-at-kore-videre-i-ulovlige-busser-med-rode-advarselslamper',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+        search_space = traverse_obj(webpage, {find_element(tag='article')}) or webpage
 
-        entries = []
+        player_ids = traverse_obj(
+            re.findall(r'x-data="(?:video_player|simple_player)\(({[^"]+})', search_space),
+            (..., {js_to_json}, {json.loads}, ('jwpMediaId', 'videoId'), {str}))
 
-        def add_entry(partner_id, kaltura_id):
-            entries.append(self.url_result(
-                f'kaltura:{partner_id}:{kaltura_id}', 'Kaltura',
-                video_id=kaltura_id))
-
-        for video_el in re.findall(r'(?s)<[^>]+\bdata-entryid\s*=[^>]*>', webpage):
-            video = extract_attributes(video_el)
-            kaltura_id = video.get('data-entryid')
-            if not kaltura_id:
-                continue
-            partner_id = video.get('data-partnerid')
-            if not partner_id:
-                continue
-            add_entry(partner_id, kaltura_id)
-        if not entries:
-            kaltura_id = self._search_regex(
-                (r'entry_id\s*:\s*["\']([0-9a-z_]+)',
-                 r'\\u002FentryId\\u002F(\w+)\\u002F'), webpage, 'kaltura id')
-            partner_id = self._search_regex(
-                (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
-                'partner id')
-            add_entry(partner_id, kaltura_id)
-        if len(entries) == 1:
-            return entries[0]
-        return self.playlist_result(entries)
+        return self.playlist_from_matches(
+            player_ids, video_id, getter=lambda x: f'jwplatform:{x}', ie=JWPlatformIE)
 
 
 class TV2DKBornholmPlayIE(InfoExtractor):

yt_dlp/extractor/twitch.py

@@ -1225,8 +1225,8 @@ class TwitchClipsIE(TwitchBaseIE):
                 'channel_id': ('broadcaster', 'id', {str}),
                 'channel_follower_count': ('broadcaster', 'followers', 'totalCount', {int_or_none}),
                 'channel_is_verified': ('broadcaster', 'isPartner', {bool}),
-                'uploader': ('broadcaster', 'displayName', {str}),
-                'uploader_id': ('broadcaster', 'id', {str}),
+                'uploader': ('curator', 'displayName', {str}),
+                'uploader_id': ('curator', 'id', {str}),
                 'categories': ('game', 'displayName', {str}, filter, all, filter),
             }),
         }