update from upstream 2022-12-01 UTC+8

1 files changed, 124 insertions, 90 deletions

diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index 961734345..97b99fc50 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -2,6 +2,7 @@ import base64
 import functools
 import re
 import itertools
+import urllib.error
 
 from .common import InfoExtractor
 from ..compat import (
@@ -30,7 +31,6 @@ from ..utils import (
     unsmuggle_url,
     urlencode_postdata,
     urljoin,
-    unescapeHTML,
     urlhandle_detect_ext,
 )
 
@@ -124,11 +124,6 @@ class VimeoBaseInfoExtractor(InfoExtractor):
     def _set_vimeo_cookie(self, name, value):
         self._set_cookie('vimeo.com', name, value)
 
-    def _vimeo_sort_formats(self, formats):
-        # Note: Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
-        # at the same time without actual units specified.
-        self._sort_formats(formats, ('quality', 'res', 'fps', 'hdr:12', 'source'))
-
     def _parse_config(self, config, video_id):
         video_data = config['video']
         video_title = video_data.get('title')
@@ -243,6 +238,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'formats': formats,
             'subtitles': subtitles,
             'is_live': is_live,
+            # Note: Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
+            # at the same time without actual units specified.
+            '_format_sort_fields': ('quality', 'res', 'fps', 'hdr:12', 'source'),
         }
 
     def _extract_original_format(self, url, video_id, unlisted_hash=None):
@@ -314,7 +312,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             )
                             \.
                         )?
-                        vimeo(?:pro)?\.com/
+                        vimeo\.com/
                         (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
                         (?:[^/]+/)*?
                         (?:
@@ -328,6 +326,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
                         /?(?:[?&].*)?(?:[#].*)?$
                     '''
     IE_NAME = 'vimeo'
+    _EMBED_REGEX = [
+        # iframe
+        r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/\d+.*?)\1',
+        # Embedded (swf embed) Vimeo player
+        r'<embed[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)\1',
+        # Non-standard embedded Vimeo player
+        r'<video[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/[0-9]+)\1',
+    ]
     _TESTS = [
         {
             'url': 'http://vimeo.com/56015672#at=0',
@@ -351,31 +357,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'skip': 'No longer available'
         },
         {
-            'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
-            'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
-            'note': 'Vimeo Pro video (#1197)',
-            'info_dict': {
-                'id': '68093876',
-                'ext': 'mp4',
-                'uploader_url': r're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
-                'uploader_id': 'openstreetmapus',
-                'uploader': 'OpenStreetMap US',
-                'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
-                'description': 'md5:2c362968038d4499f4d79f88458590c1',
-                'duration': 1595,
-                'upload_date': '20130610',
-                'timestamp': 1370893156,
-                'license': 'by',
-                'thumbnail': 'https://i.vimeocdn.com/video/440260469-19b0d92fca3bd84066623b53f1eb8aaa3980c6c809e2d67b6b39ab7b4a77a344-d_960',
-                'view_count': int,
-                'comment_count': int,
-                'like_count': int,
-            },
-            'params': {
-                'format': 'best[protocol=https]',
-            },
-        },
-        {
             'url': 'http://player.vimeo.com/video/54469442',
             'md5': 'b3e7f4d2cbb53bd7dc3bb6ff4ed5cfbd',
             'note': 'Videos that embed the url in the player page',
@@ -729,29 +710,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
         # vimeo embed with check-password page protected by Referer header
     ]
 
-    @staticmethod
-    def _extract_urls(url, webpage):
-        urls = []
-        # Look for embedded (iframe) Vimeo player
-        for mobj in re.finditer(
-                r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/\d+.*?)\1',
-                webpage):
-            urls.append(VimeoIE._smuggle_referrer(unescapeHTML(mobj.group('url')), url))
-        PLAIN_EMBED_RE = (
-            # Look for embedded (swf embed) Vimeo player
-            r'<embed[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)\1',
-            # Look more for non-standard embedded Vimeo player
-            r'<video[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?vimeo\.com/[0-9]+)\1',
-        )
-        for embed_re in PLAIN_EMBED_RE:
-            for mobj in re.finditer(embed_re, webpage):
-                urls.append(mobj.group('url'))
-        return urls
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        for embed_url in super()._extract_embed_urls(url, webpage):
+            yield cls._smuggle_referrer(embed_url, url)
 
-    @staticmethod
-    def _extract_url(url, webpage):
-        urls = VimeoIE._extract_urls(url, webpage)
-        return urls[0] if urls else None
+    @classmethod
+    def _extract_url(cls, url, webpage):
+        return next(cls._extract_embed_urls(url, webpage), None)
 
     def _verify_player_video_password(self, url, video_id, headers):
         password = self._get_video_password()
@@ -784,7 +750,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             })
         info = self._parse_config(self._download_json(
             video['config_url'], video_id), video_id)
-        self._vimeo_sort_formats(info['formats'])
         get_timestamp = lambda x: parse_iso8601(video.get(x + '_time'))
         info.update({
             'description': video.get('description'),
@@ -848,15 +813,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if unlisted_hash:
             return self._extract_from_api(video_id, unlisted_hash)
 
-        orig_url = url
-        is_pro = 'vimeopro.com/' in url
-        if is_pro:
-            # some videos require portfolio_id to be present in player url
-            # https://github.com/ytdl-org/youtube-dl/issues/20070
-            url = self._extract_url(url, self._download_webpage(url, video_id))
-            if not url:
-                url = 'https://vimeo.com/' + video_id
-        elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
+        if any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
             url = 'https://vimeo.com/' + video_id
 
         self._try_album_password(url)
@@ -878,13 +835,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         if '://player.vimeo.com/video/' in url:
             config = self._parse_json(self._search_regex(
-                r'\bconfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
+                r'\b(?:playerC|c)onfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)
-            info = self._parse_config(config, video_id)
-            self._vimeo_sort_formats(info['formats'])
-            return info
+            return self._parse_config(config, video_id)
 
         if re.search(r'<form[^>]+?id="pw_form"', webpage):
             video_password = self._get_video_password()
@@ -960,14 +915,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
             video_description = self._html_search_meta(
                 ['description', 'og:description', 'twitter:description'],
                 webpage, default=None)
-        if not video_description and is_pro:
-            orig_webpage = self._download_webpage(
-                orig_url, video_id,
-                note='Downloading webpage for description',
-                fatal=False)
-            if orig_webpage:
-                video_description = self._html_search_meta(
-                    'description', orig_webpage, default=None)
         if not video_description:
             self.report_warning('Cannot find video description')
 
@@ -989,7 +936,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         info_dict_config = self._parse_config(config, video_id)
         formats.extend(info_dict_config['formats'])
-        self._vimeo_sort_formats(formats)
+        info_dict['_format_sort_fields'] = info_dict_config['_format_sort_fields']
 
         json_ld = self._search_json_ld(webpage, video_id, default={})
 
@@ -1012,7 +959,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         return merge_dicts(info_dict, info_dict_config, json_ld)
 
 
-class VimeoOndemandIE(VimeoIE):
+class VimeoOndemandIE(VimeoIE):  # XXX: Do not subclass from concrete IE
     IE_NAME = 'vimeo:ondemand'
     _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?:[^/]+/)?(?P<id>[^/?#&]+)'
     _TESTS = [{
@@ -1137,9 +1084,9 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
         return self._extract_videos(channel_id, self._BASE_URL_TEMPL % channel_id)
 
 
-class VimeoUserIE(VimeoChannelIE):
+class VimeoUserIE(VimeoChannelIE):  # XXX: Do not subclass from concrete IE
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<id>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?!(?:[0-9]+|watchlater)(?:$|[?#/]))(?P<id>[^/]+)(?:/videos)?/?(?:$|[?#])'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
         'url': 'https://vimeo.com/nkistudio/videos',
@@ -1148,6 +1095,9 @@ class VimeoUserIE(VimeoChannelIE):
             'id': 'nkistudio',
         },
         'playlist_mincount': 66,
+    }, {
+        'url': 'https://vimeo.com/nkistudio/',
+        'only_matching': True,
     }]
     _BASE_URL_TEMPL = 'https://vimeo.com/%s'
 
@@ -1244,7 +1194,7 @@ class VimeoAlbumIE(VimeoBaseInfoExtractor):
             entries, album_id, album.get('name'), album.get('description'))
 
 
-class VimeoGroupsIE(VimeoChannelIE):
+class VimeoGroupsIE(VimeoChannelIE):  # XXX: Do not subclass from concrete IE
     IE_NAME = 'vimeo:group'
     _VALID_URL = r'https://vimeo\.com/groups/(?P<id>[^/]+)(?:/(?!videos?/\d+)|$)'
     _TESTS = [{
@@ -1331,12 +1281,11 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
             page_url + '/action', video_id)
         if source_format:
             info_dict['formats'].append(source_format)
-        self._vimeo_sort_formats(info_dict['formats'])
         info_dict['description'] = clean_html(clip_data.get('description'))
         return info_dict
 
 
-class VimeoWatchLaterIE(VimeoChannelIE):
+class VimeoWatchLaterIE(VimeoChannelIE):  # XXX: Do not subclass from concrete IE
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, ":vimeowatchlater" keyword (requires authentication)'
     _VALID_URL = r'https://vimeo\.com/(?:home/)?watchlater|:vimeowatchlater'
@@ -1359,7 +1308,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
         return self._extract_videos('watchlater', 'https://vimeo.com/watchlater')
 
 
-class VimeoLikesIE(VimeoChannelIE):
+class VimeoLikesIE(VimeoChannelIE):  # XXX: Do not subclass from concrete IE
     _VALID_URL = r'https://(?:www\.)?vimeo\.com/(?P<id>[^/]+)/likes/?(?:$|[?#]|sort:)'
     IE_NAME = 'vimeo:likes'
     IE_DESC = 'Vimeo user likes'
@@ -1386,12 +1335,12 @@ class VimeoLikesIE(VimeoChannelIE):
 class VHXEmbedIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vhx:embed'
     _VALID_URL = r'https?://embed\.vhx\.tv/videos/(?P<id>\d+)'
+    _EMBED_REGEX = [r'<iframe[^>]+src="(?P<url>https?://embed\.vhx\.tv/videos/\d+[^"]*)"']
 
-    @staticmethod
-    def _extract_url(url, webpage):
-        mobj = re.search(
-            r'<iframe[^>]+src="(https?://embed\.vhx\.tv/videos/\d+[^"]*)"', webpage)
-        return VimeoIE._smuggle_referrer(unescapeHTML(mobj.group(1)), url) if mobj else None
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        for embed_url in super()._extract_embed_urls(url, webpage):
+            yield cls._smuggle_referrer(embed_url, url)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -1403,5 +1352,90 @@ class VHXEmbedIE(VimeoBaseInfoExtractor):
         config = self._download_json(config_url, video_id)
         info = self._parse_config(config, video_id)
         info['id'] = video_id
-        self._vimeo_sort_formats(info['formats'])
         return info
+
+
+class VimeoProIE(VimeoBaseInfoExtractor):
+    IE_NAME = 'vimeo:pro'
+    _VALID_URL = r'https?://(?:www\.)?vimeopro\.com/[^/?#]+/(?P<slug>[^/?#]+)(?:(?:/videos?/(?P<id>[0-9]+)))?'
+    _TESTS = [{
+        # Vimeo URL derived from video_id
+        'url': 'http://vimeopro.com/openstreetmapus/state-of-the-map-us-2013/video/68093876',
+        'md5': '3b5ca6aa22b60dfeeadf50b72e44ed82',
+        'note': 'Vimeo Pro video (#1197)',
+        'info_dict': {
+            'id': '68093876',
+            'ext': 'mp4',
+            'uploader_url': r're:https?://(?:www\.)?vimeo\.com/openstreetmapus',
+            'uploader_id': 'openstreetmapus',
+            'uploader': 'OpenStreetMap US',
+            'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
+            'description': 'md5:2c362968038d4499f4d79f88458590c1',
+            'duration': 1595,
+            'upload_date': '20130610',
+            'timestamp': 1370893156,
+            'license': 'by',
+            'thumbnail': 'https://i.vimeocdn.com/video/440260469-19b0d92fca3bd84066623b53f1eb8aaa3980c6c809e2d67b6b39ab7b4a77a344-d_960',
+            'view_count': int,
+            'comment_count': int,
+            'like_count': int,
+            'tags': 'count:1',
+        },
+        'params': {
+            'format': 'best[protocol=https]',
+        },
+    }, {
+        # password-protected VimeoPro page with Vimeo player embed
+        'url': 'https://vimeopro.com/cadfem/simulation-conference-mechanische-systeme-in-perfektion',
+        'info_dict': {
+            'id': '764543723',
+            'ext': 'mp4',
+            'title': 'Mechanische Systeme in Perfektion: Realität erfassen, Innovation treiben',
+            'thumbnail': 'https://i.vimeocdn.com/video/1543784598-a1a750494a485e601110136b9fe11e28c2131942452b3a5d30391cb3800ca8fd-d_1280',
+            'description': 'md5:2a9d195cd1b0f6f79827107dc88c2420',
+            'uploader': 'CADFEM',
+            'uploader_id': 'cadfem',
+            'uploader_url': 'https://vimeo.com/cadfem',
+            'duration': 12505,
+            'chapters': 'count:10',
+        },
+        'params': {
+            'videopassword': 'Conference2022',
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id, video_id = self._match_valid_url(url).group('slug', 'id')
+        if video_id:
+            display_id = video_id
+        webpage = self._download_webpage(url, display_id)
+
+        password_form = self._search_regex(
+            r'(?is)<form[^>]+?method=["\']post["\'][^>]*>(.+?password.+?)</form>',
+            webpage, 'password form', default=None)
+        if password_form:
+            try:
+                webpage = self._download_webpage(url, display_id, data=urlencode_postdata({
+                    'password': self._get_video_password(),
+                    **self._hidden_inputs(password_form),
+                }), note='Logging in with video password')
+            except ExtractorError as e:
+                if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 418:
+                    raise ExtractorError('Wrong video password', expected=True)
+                raise
+
+        description = None
+        # even if we have video_id, some videos require player URL with portfolio_id query param
+        # https://github.com/ytdl-org/youtube-dl/issues/20070
+        vimeo_url = VimeoIE._extract_url(url, webpage)
+        if vimeo_url:
+            description = self._html_search_meta('description', webpage, default=None)
+        elif video_id:
+            vimeo_url = f'https://vimeo.com/{video_id}'
+        else:
+            raise ExtractorError(
+                'No Vimeo embed or video ID could be found in VimeoPro page', expected=True)
+
+        return self.url_result(vimeo_url, VimeoIE, video_id, url_transparent=True,
+                               description=description)