[ie/GetCourseRu] Fix extractors (#12943)

Closes #12941
Authored by: troex

yt_dlp/extractor/cda.py

@@ -13,16 +13,17 @@ from ..compat import compat_ord
 from ..utils import (
     ExtractorError,
     OnDemandPagedList,
+    determine_ext,
     float_or_none,
     int_or_none,
     merge_dicts,
     multipart_encode,
     parse_duration,
-    traverse_obj,
     try_call,
-    try_get,
+    url_or_none,
     urljoin,
 )
+from ..utils.traversal import traverse_obj
 
 
 class CDAIE(InfoExtractor):
@@ -290,34 +291,47 @@ class CDAIE(InfoExtractor):
             if not video or 'file' not in video:
                 self.report_warning(f'Unable to extract {version} version information')
                 return
-            if video['file'].startswith('uggc'):
-                video['file'] = codecs.decode(video['file'], 'rot_13')
-                if video['file'].endswith('adc.mp4'):
-                    video['file'] = video['file'].replace('adc.mp4', '.mp4')
-            elif not video['file'].startswith('http'):
-                video['file'] = decrypt_file(video['file'])
             video_quality = video.get('quality')
             qualities = video.get('qualities', {})
             video_quality = next((k for k, v in qualities.items() if v == video_quality), video_quality)
-            info_dict['formats'].append({
-                'url': video['file'],
-                'format_id': video_quality,
-                'height': int_or_none(video_quality[:-1]),
-            })
+            if video.get('file'):
+                if video['file'].startswith('uggc'):
+                    video['file'] = codecs.decode(video['file'], 'rot_13')
+                    if video['file'].endswith('adc.mp4'):
+                        video['file'] = video['file'].replace('adc.mp4', '.mp4')
+                elif not video['file'].startswith('http'):
+                    video['file'] = decrypt_file(video['file'])
+                info_dict['formats'].append({
+                    'url': video['file'],
+                    'format_id': video_quality,
+                    'height': int_or_none(video_quality[:-1]),
+                })
             for quality, cda_quality in qualities.items():
                 if quality == video_quality:
                     continue
                 data = {'jsonrpc': '2.0', 'method': 'videoGetLink', 'id': 2,
                         'params': [video_id, cda_quality, video.get('ts'), video.get('hash2'), {}]}
                 data = json.dumps(data).encode()
-                video_url = self._download_json(
+                response = self._download_json(
                     f'https://www.cda.pl/video/{video_id}', video_id, headers={
                         'Content-Type': 'application/json',
                         'X-Requested-With': 'XMLHttpRequest',
                     }, data=data, note=f'Fetching {quality} url',
                     errnote=f'Failed to fetch {quality} url', fatal=False)
-                if try_get(video_url, lambda x: x['result']['status']) == 'ok':
-                    video_url = try_get(video_url, lambda x: x['result']['resp'])
+                if (
+                    traverse_obj(response, ('result', 'status')) != 'ok'
+                    or not traverse_obj(response, ('result', 'resp', {url_or_none}))
+                ):
+                    continue
+                video_url = response['result']['resp']
+                ext = determine_ext(video_url)
+                if ext == 'mpd':
+                    info_dict['formats'].extend(self._extract_mpd_formats(
+                        video_url, video_id, mpd_id='dash', fatal=False))
+                elif ext == 'm3u8':
+                    info_dict['formats'].extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', m3u8_id='hls', fatal=False))
+                else:
                     info_dict['formats'].append({
                         'url': video_url,
                         'format_id': quality,

yt_dlp/extractor/generic.py

@@ -16,7 +16,6 @@ from ..utils import (
     MEDIA_EXTENSIONS,
     ExtractorError,
     UnsupportedError,
-    base_url,
     determine_ext,
     determine_protocol,
     dict_get,
@@ -38,6 +37,7 @@ from ..utils import (
     unescapeHTML,
     unified_timestamp,
     unsmuggle_url,
+    update_url,
     update_url_query,
     url_or_none,
     urlhandle_detect_ext,
@@ -2538,12 +2538,13 @@ class GenericIE(InfoExtractor):
                 return self.playlist_result(
                     self._parse_xspf(
                         doc, video_id, xspf_url=url,
-                        xspf_base_url=full_response.url),
+                        xspf_base_url=new_url),
                     video_id)
             elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
                 info_dict['formats'], info_dict['subtitles'] = self._parse_mpd_formats_and_subtitles(
                     doc,
-                    mpd_base_url=base_url(full_response.url),
+                    # Do not use yt_dlp.utils.base_url here since it will raise on file:// URLs
+                    mpd_base_url=update_url(new_url, query=None, fragment=None).rpartition('/')[0],
                     mpd_url=url)
                 info_dict['live_status'] = 'is_live' if doc.get('type') == 'dynamic' else None
                 self._extra_manifest_info(info_dict, url)

yt_dlp/extractor/getcourseru.py

@@ -8,7 +8,7 @@ from ..utils.traversal import traverse_obj
 
 
 class GetCourseRuPlayerIE(InfoExtractor):
-    _VALID_URL = r'https?://player02\.getcourse\.ru/sign-player/?\?(?:[^#]+&)?json=[^#&]+'
+    _VALID_URL = r'https?://(?:player02\.getcourse\.ru|cf-api-2\.vhcdn\.com)/sign-player/?\?(?:[^#]+&)?json=[^#&]+'
     _EMBED_REGEX = [rf'<iframe[^>]+\bsrc=[\'"](?P<url>{_VALID_URL}[^\'"]*)']
     _TESTS = [{
         'url': 'http://player02.getcourse.ru/sign-player/?json=eyJ2aWRlb19oYXNoIjoiMTkwYmRmOTNmMWIyOTczNTMwOTg1M2E3YTE5ZTI0YjMiLCJ1c2VyX2lkIjozNTk1MjUxODMsInN1Yl9sb2dpbl91c2VyX2lkIjpudWxsLCJsZXNzb25faWQiOm51bGwsImlwIjoiNDYuMTQyLjE4Mi4yNDciLCJnY19ob3N0IjoiYWNhZGVteW1lbC5vbmxpbmUiLCJ0aW1lIjoxNzA1NDQ5NjQyLCJwYXlsb2FkIjoidV8zNTk1MjUxODMiLCJ1aV9sYW5ndWFnZSI6InJ1IiwiaXNfaGF2ZV9jdXN0b21fc3R5bGUiOnRydWV9&s=354ad2c993d95d5ac629e3133d6cefea&vh-static-feature=zigzag',
@@ -20,6 +20,16 @@ class GetCourseRuPlayerIE(InfoExtractor):
             'duration': 1693,
         },
         'skip': 'JWT expired',
+    }, {
+        'url': 'https://cf-api-2.vhcdn.com/sign-player/?json=example',
+        'info_dict': {
+            'id': '435735291',
+            'title': '8afd7c489952108e00f019590f3711f3',
+            'ext': 'mp4',
+            'thumbnail': 'https://preview-htz.vhcdn.com/preview/8afd7c489952108e00f019590f3711f3/preview.jpg?version=1682170973&host=vh-72',
+            'duration': 777,
+        },
+        'skip': 'JWT expired',
     }]
 
     def _real_extract(self, url):
@@ -168,7 +178,7 @@ class GetCourseRuIE(InfoExtractor):
 
         playlist_id = self._search_regex(
             r'window\.(?:lessonId|gcsObjectId)\s*=\s*(\d+)', webpage, 'playlist id', default=display_id)
-        title = self._og_search_title(webpage) or self._html_extract_title(webpage)
+        title = self._og_search_title(webpage, default=None) or self._html_extract_title(webpage)
 
         return self.playlist_from_matches(
             re.findall(GetCourseRuPlayerIE._EMBED_REGEX[0], webpage),

yt_dlp/extractor/twitcasting.py

@@ -14,12 +14,13 @@ from ..utils import (
     parse_duration,
     qualities,
     str_to_int,
-    traverse_obj,
     try_get,
     unified_timestamp,
+    url_or_none,
     urlencode_postdata,
     urljoin,
 )
+from ..utils.traversal import traverse_obj
 
 
 class TwitCastingIE(InfoExtractor):
@@ -138,13 +139,7 @@ class TwitCastingIE(InfoExtractor):
             r'data-toggle="true"[^>]+datetime="([^"]+)"',
             webpage, 'datetime', None))
 
-        stream_server_data = self._download_json(
-            f'https://twitcasting.tv/streamserver.php?target={uploader_id}&mode=client', video_id,
-            'Downloading live info', fatal=False)
-
         is_live = any(f'data-{x}' in webpage for x in ['is-onlive="true"', 'live-type="live"', 'status="online"'])
-        if not traverse_obj(stream_server_data, 'llfmp4') and is_live:
-            self.raise_login_required(method='cookies')
 
         base_dict = {
             'title': title,
@@ -165,28 +160,37 @@ class TwitCastingIE(InfoExtractor):
                 return [data_movie_url]
 
         m3u8_urls = (try_get(webpage, find_dmu, list)
-                     or traverse_obj(video_js_data, (..., 'source', 'url'))
-                     or ([f'https://twitcasting.tv/{uploader_id}/metastream.m3u8'] if is_live else None))
-        if not m3u8_urls:
-            raise ExtractorError('Failed to get m3u8 playlist')
+                     or traverse_obj(video_js_data, (..., 'source', 'url')))
 
         if is_live:
-            m3u8_url = m3u8_urls[0]
-            formats = self._extract_m3u8_formats(
-                m3u8_url, video_id, ext='mp4', m3u8_id='hls',
-                live=True, headers=self._M3U8_HEADERS)
+            stream_data = self._download_json(
+                'https://twitcasting.tv/streamserver.php',
+                video_id, 'Downloading live info', query={
+                    'target': uploader_id,
+                    'mode': 'client',
+                    'player': 'pc_web',
+                })
 
-            if traverse_obj(stream_server_data, ('hls', 'source')):
-                formats.extend(self._extract_m3u8_formats(
-                    m3u8_url, video_id, ext='mp4', m3u8_id='source',
-                    live=True, query={'mode': 'source'},
-                    note='Downloading source quality m3u8',
-                    headers=self._M3U8_HEADERS, fatal=False))
+            formats = []
+            # low: 640x360, medium: 1280x720, high: 1920x1080
+            qq = qualities(['low', 'medium', 'high'])
+            for quality, m3u8_url in traverse_obj(stream_data, (
+                'tc-hls', 'streams', {dict.items}, lambda _, v: url_or_none(v[1]),
+            )):
+                formats.append({
+                    'url': m3u8_url,
+                    'format_id': f'hls-{quality}',
+                    'ext': 'mp4',
+                    'quality': qq(quality),
+                    'protocol': 'm3u8',
+                    'http_headers': self._M3U8_HEADERS,
+                })
 
             if websockets:
                 qq = qualities(['base', 'mobilesource', 'main'])
-                streams = traverse_obj(stream_server_data, ('llfmp4', 'streams')) or {}
-                for mode, ws_url in streams.items():
+                for mode, ws_url in traverse_obj(stream_data, (
+                    'llfmp4', 'streams', {dict.items}, lambda _, v: url_or_none(v[1]),
+                )):
                     formats.append({
                         'url': ws_url,
                         'format_id': f'ws-{mode}',
@@ -197,10 +201,15 @@ class TwitCastingIE(InfoExtractor):
                         'protocol': 'websocket_frag',
                     })
 
+            if not formats:
+                self.raise_login_required()
+
             infodict = {
                 'formats': formats,
                 '_format_sort_fields': ('source', ),
             }
+        elif not m3u8_urls:
+            raise ExtractorError('Failed to get m3u8 playlist')
         elif len(m3u8_urls) == 1:
             formats = self._extract_m3u8_formats(
                 m3u8_urls[0], video_id, 'mp4', headers=self._M3U8_HEADERS)

yt_dlp/extractor/vimeo.py

@@ -244,9 +244,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             join_nonempty(f'https://api.vimeo.com/videos/{video_id}', unlisted_hash, delim=':'),
             video_id, 'Downloading API JSON', headers={
                 'Authorization': f'jwt {jwt_token}',
-                'Accept': 'application/json',
+                'Accept': 'application/vnd.vimeo.*+json;version=3.4.10',
             }, query={
+                # TODO: Reverse-engineer generating the 'anon_signature' param
+                # Ref: https://f.vimeocdn.com/js_opt/app/vimeo-next/_next/static/chunks/60908-af70235e46909bce.js
+                'outro': 'beginning',  # Needed to avoid https://github.com/yt-dlp/yt-dlp/issues/12974
                 'fields': ','.join((
+                    # 'embed_player_config_url' is a viable alternative to 'config_url'
                     'config_url', 'created_time', 'description', 'download', 'license',
                     'metadata.connections.comments.total', 'metadata.connections.likes.total',
                     'release_time', 'stats.plays')),
@@ -410,6 +414,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'duration': 10,
                 'comment_count': int,
                 'like_count': int,
+                'view_count': int,
                 'thumbnail': 'https://i.vimeocdn.com/video/440665496-b2c5aee2b61089442c794f64113a8e8f7d5763c3e6b3ebfaf696ae6413f8b1f4-d',
             },
             'params': {
@@ -500,15 +505,16 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader': 'The DMCI',
                 'uploader_url': r're:https?://(?:www\.)?vimeo\.com/dmci',
                 'uploader_id': 'dmci',
-                'timestamp': 1324343742,
+                'timestamp': 1324361742,
                 'upload_date': '20111220',
-                'description': 'md5:ae23671e82d05415868f7ad1aec21147',
+                'description': 'md5:f37b4ad0f3ded6fa16f38ecde16c3c44',
                 'duration': 60,
                 'comment_count': int,
                 'view_count': int,
                 'thumbnail': 'https://i.vimeocdn.com/video/231174622-dd07f015e9221ff529d451e1cc31c982b5d87bfafa48c4189b1da72824ee289a-d',
                 'like_count': int,
-                'tags': 'count:11',
+                'release_timestamp': 1324361742,
+                'release_date': '20111220',
             },
             # 'params': {'format': 'Original'},
             'expected_warnings': ['Failed to parse XML: not well-formed'],
@@ -521,15 +527,18 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'id': '393756517',
                 # 'ext': 'mov',
                 'ext': 'mp4',
-                'timestamp': 1582642091,
+                'timestamp': 1582660091,
                 'uploader_id': 'frameworkla',
                 'title': 'Straight To Hell - Sabrina: Netflix',
                 'uploader': 'Framework Studio',
-                'description': 'md5:f2edc61af3ea7a5592681ddbb683db73',
                 'upload_date': '20200225',
                 'duration': 176,
                 'thumbnail': 'https://i.vimeocdn.com/video/859377297-836494a4ef775e9d4edbace83937d9ad34dc846c688c0c419c0e87f7ab06c4b3-d',
                 'uploader_url': 'https://vimeo.com/frameworkla',
+                'comment_count': int,
+                'like_count': int,
+                'release_timestamp': 1582660091,
+                'release_date': '20200225',
             },
             # 'params': {'format': 'source'},
             'expected_warnings': ['Failed to parse XML: not well-formed'],
@@ -630,7 +639,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'description': str,  # FIXME: Dynamic SEO spam description
                 'upload_date': '20150209',
                 'timestamp': 1423518307,
-                'thumbnail': 'https://i.vimeocdn.com/video/default',
+                'thumbnail': r're:https://i\.vimeocdn\.com/video/default',
                 'duration': 10,
                 'like_count': int,
                 'uploader_url': 'https://vimeo.com/user20132939',
@@ -667,6 +676,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'like_count': int,
                 'uploader_url': 'https://vimeo.com/aliniamedia',
                 'release_date': '20160329',
+                'view_count': int,
             },
             'params': {'skip_download': True},
             'expected_warnings': ['Failed to parse XML: not well-formed'],
@@ -678,18 +688,19 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 # 'ext': 'm4v',
                 'ext': 'mp4',
                 'title': 'Eastnor Castle 2015 Firework Champions - The Promo!',
-                'description': 'md5:5967e090768a831488f6e74b7821b3c1',
+                'description': 'md5:9441e6829ae94f380cc6417d982f63ac',
                 'uploader_id': 'fireworkchampions',
                 'uploader': 'Firework Champions',
                 'upload_date': '20150910',
-                'timestamp': 1441901895,
+                'timestamp': 1441916295,
                 'thumbnail': 'https://i.vimeocdn.com/video/534715882-6ff8e4660cbf2fea68282876d8d44f318825dfe572cc4016e73b3266eac8ae3a-d',
                 'uploader_url': 'https://vimeo.com/fireworkchampions',
-                'tags': 'count:6',
                 'duration': 229,
                 'view_count': int,
                 'like_count': int,
                 'comment_count': int,
+                'release_timestamp': 1441916295,
+                'release_date': '20150910',
             },
             'params': {
                 'skip_download': True,
@@ -820,7 +831,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader': 'Raja Virdi',
                 'uploader_id': 'rajavirdi',
                 'uploader_url': 'https://vimeo.com/rajavirdi',
-                'duration': 309,
+                'duration': 300,
                 'thumbnail': r're:https://i\.vimeocdn\.com/video/1716727772-[\da-f]+-d',
             },
             # 'params': {'format': 'source'},
@@ -1122,7 +1133,7 @@ class VimeoOndemandIE(VimeoIE):  # XXX: Do not subclass from concrete IE
             'description': 'md5:aeeba3dbd4d04b0fa98a4fdc9c639998',
             'upload_date': '20140906',
             'timestamp': 1410032453,
-            'thumbnail': 'https://i.vimeocdn.com/video/488238335-d7bf151c364cff8d467f1b73784668fe60aae28a54573a35d53a1210ae283bd8-d_1280',
+            'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d',
             'comment_count': int,
             'license': 'https://creativecommons.org/licenses/by-nc-nd/3.0/',
             'duration': 53,
@@ -1132,7 +1143,7 @@ class VimeoOndemandIE(VimeoIE):  # XXX: Do not subclass from concrete IE
         'params': {
             'format': 'best[protocol=https]',
         },
-        'expected_warnings': ['Unable to download JSON metadata'],
+        'expected_warnings': ['Failed to parse XML: not well-formed'],
     }, {
         # requires Referer to be passed along with og:video:url
         'url': 'https://vimeo.com/ondemand/36938/126682985',
@@ -1149,13 +1160,14 @@ class VimeoOndemandIE(VimeoIE):  # XXX: Do not subclass from concrete IE
             'duration': 121,
             'comment_count': int,
             'view_count': int,
-            'thumbnail': 'https://i.vimeocdn.com/video/517077723-7066ae1d9a79d3eb361334fb5d58ec13c8f04b52f8dd5eadfbd6fb0bcf11f613-d_1280',
+            'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d',
             'like_count': int,
+            'tags': 'count:5',
         },
         'params': {
             'skip_download': True,
         },
-        'expected_warnings': ['Unable to download JSON metadata'],
+        'expected_warnings': ['Failed to parse XML: not well-formed'],
     }, {
         'url': 'https://vimeo.com/ondemand/nazmaalik',
         'only_matching': True,
@@ -1237,7 +1249,7 @@ class VimeoUserIE(VimeoChannelIE):  # XXX: Do not subclass from concrete IE
     _TESTS = [{
         'url': 'https://vimeo.com/nkistudio/videos',
         'info_dict': {
-            'title': 'Nki',
+            'title': 'AKAMA',
             'id': 'nkistudio',
         },
         'playlist_mincount': 66,
@@ -1370,10 +1382,10 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
             'uploader_id': 'user170863801',
             'uploader_url': 'https://vimeo.com/user170863801',
             'duration': 30,
-            'thumbnail': 'https://i.vimeocdn.com/video/1912612821-09a43bd2e75c203d503aed89de7534f28fc4474a48f59c51999716931a246af5-d_1280',
+            'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d',
         },
         'params': {'skip_download': 'm3u8'},
-        'expected_warnings': ['Failed to parse XML'],
+        'expected_warnings': ['Failed to parse XML: not well-formed'],
     }, {
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
@@ -1528,20 +1540,22 @@ class VimeoProIE(VimeoBaseInfoExtractor):
             'uploader_id': 'openstreetmapus',
             'uploader': 'OpenStreetMap US',
             'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
-            'description': 'md5:2c362968038d4499f4d79f88458590c1',
+            'description': 'md5:8cf69a1a435f2d763f4adf601e9c3125',
             'duration': 1595,
             'upload_date': '20130610',
-            'timestamp': 1370893156,
+            'timestamp': 1370907556,
             'license': 'by',
-            'thumbnail': 'https://i.vimeocdn.com/video/440260469-19b0d92fca3bd84066623b53f1eb8aaa3980c6c809e2d67b6b39ab7b4a77a344-d_960',
+            'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d',
             'view_count': int,
             'comment_count': int,
             'like_count': int,
-            'tags': 'count:1',
+            'release_timestamp': 1370907556,
+            'release_date': '20130610',
         },
         'params': {
             'format': 'best[protocol=https]',
         },
+        'expected_warnings': ['Failed to parse XML: not well-formed'],
     }, {
         # password-protected VimeoPro page with Vimeo player embed
         'url': 'https://vimeopro.com/cadfem/simulation-conference-mechanische-systeme-in-perfektion',
@@ -1549,7 +1563,7 @@ class VimeoProIE(VimeoBaseInfoExtractor):
             'id': '764543723',
             'ext': 'mp4',
             'title': 'Mechanische Systeme in Perfektion: Realität erfassen, Innovation treiben',
-            'thumbnail': 'https://i.vimeocdn.com/video/1543784598-a1a750494a485e601110136b9fe11e28c2131942452b3a5d30391cb3800ca8fd-d_1280',
+            'thumbnail': r're:https://i\.vimeocdn\.com/video/\d+-[\da-f]+-d',
             'description': 'md5:2a9d195cd1b0f6f79827107dc88c2420',
             'uploader': 'CADFEM',
             'uploader_id': 'cadfem',
@@ -1561,6 +1575,7 @@ class VimeoProIE(VimeoBaseInfoExtractor):
             'videopassword': 'Conference2022',
             'skip_download': True,
         },
+        'expected_warnings': ['Failed to parse XML: not well-formed'],
     }]
 
     def _real_extract(self, url):