diff options
Diffstat (limited to 'hypervideo_dl/extractor/kaltura.py')
-rw-r--r-- | hypervideo_dl/extractor/kaltura.py | 265 |
1 files changed, 215 insertions, 50 deletions
diff --git a/hypervideo_dl/extractor/kaltura.py b/hypervideo_dl/extractor/kaltura.py index f6dfc9c..95e2dee 100644 --- a/hypervideo_dl/extractor/kaltura.py +++ b/hypervideo_dl/extractor/kaltura.py @@ -1,8 +1,6 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import re import base64 +import json +import re from .common import InfoExtractor from ..compat import ( @@ -16,13 +14,15 @@ from ..utils import ( int_or_none, unsmuggle_url, smuggle_url, + traverse_obj, + remove_start ) class KalturaIE(InfoExtractor): _VALID_URL = r'''(?x) (?: - kaltura:(?P<partner_id>\d+):(?P<id>[0-9a-z_]+)| + kaltura:(?P<partner_id>\w+):(?P<id>\w+)(?::(?P<player_type>\w+))?| https?:// (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com(?::\d+)?/ (?: @@ -36,7 +36,7 @@ class KalturaIE(InfoExtractor): ) ''' _SERVICE_URL = 'http://cdnapi.kaltura.com' - _SERVICE_BASE = '/api_v3/index.php' + _SERVICE_BASE = '/api_v3/service/multirequest' # See https://github.com/kaltura/server/blob/master/plugins/content/caption/base/lib/model/enums/CaptionType.php _CAPTION_TYPES = { 1: 'srt', @@ -57,6 +57,7 @@ class KalturaIE(InfoExtractor): 'thumbnail': 're:^https?://.*/thumbnail/.*', 'timestamp': int, }, + 'skip': 'The access to this service is forbidden since the specified partner is blocked' }, { 'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4', @@ -109,16 +110,85 @@ class KalturaIE(InfoExtractor): # unavailable source format 'url': 'kaltura:513551:1_66x4rg7o', 'only_matching': True, + }, + { + # html5lib URL using kwidget player + 'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.46/mwEmbedFrame.php/p/691292/uiconf_id/20499062/entry_id/0_c076mna6?wid=_691292&iframeembed=true&playerId=kaltura_player_1420508608&entry_id=0_c076mna6&flashvars%5BakamaiHD.loadingPolicy%5D=preInitialize&flashvars%5BakamaiHD.asyncInit%5D=true&flashvars%5BstreamerType%5D=hdnetwork', + 'info_dict': { + 'id': '0_c076mna6', + 'ext': 'mp4', + 'title': 'md5:4883e7acbcbf42583a2dddc97dee4855', + 'duration': 3608, + 'uploader_id': 'commons@swinburne.edu.au', + 'timestamp': 1408086874, + 'view_count': int, + 'upload_date': '20140815', + 'thumbnail': 'http://cfvod.kaltura.com/p/691292/sp/69129200/thumbnail/entry_id/0_c076mna6/version/100022', + } + }, + { + # html5lib playlist URL using kwidget player + 'url': 'https://cdnapisec.kaltura.com/html5/html5lib/v2.89/mwEmbedFrame.php/p/2019031/uiconf_id/40436601?wid=1_4j3m32cv&iframeembed=true&playerId=kaltura_player_&flashvars[playlistAPI.kpl0Id]=1_jovey5nu&flashvars[ks]=&&flashvars[imageDefaultDuration]=30&flashvars[localizationCode]=en&flashvars[leadWithHTML5]=true&flashvars[forceMobileHTML5]=true&flashvars[nextPrevBtn.plugin]=true&flashvars[hotspots.plugin]=true&flashvars[sideBarContainer.plugin]=true&flashvars[sideBarContainer.position]=left&flashvars[sideBarContainer.clickToClose]=true&flashvars[chapters.plugin]=true&flashvars[chapters.layout]=vertical&flashvars[chapters.thumbnailRotator]=false&flashvars[streamSelector.plugin]=true&flashvars[EmbedPlayer.SpinnerTarget]=videoHolder&flashvars[dualScreen.plugin]=true&flashvars[playlistAPI.playlistUrl]=https://canvasgatechtest.kaf.kaltura.com/playlist/details/{playlistAPI.kpl0Id}/categoryid/126428551', + 'info_dict': { + 'id': '1_jovey5nu', + 'title': '00-00 Introduction' + }, + 'playlist': [ + { + 'info_dict': { + 'id': '1_b1y5hlvx', + 'ext': 'mp4', + 'title': 'CS7646_00-00 Introductio_Introduction', + 'duration': 91, + 'thumbnail': 'http://cfvod.kaltura.com/p/2019031/sp/201903100/thumbnail/entry_id/1_b1y5hlvx/version/100001', + 'view_count': int, + 'timestamp': 1533154447, + 'upload_date': '20180801', + 'uploader_id': 'djoyner3', + } + }, { + 'info_dict': { + 'id': '1_jfb7mdpn', + 'ext': 'mp4', + 'title': 'CS7646_00-00 Introductio_Three parts to the course', + 'duration': 63, + 'thumbnail': 'http://cfvod.kaltura.com/p/2019031/sp/201903100/thumbnail/entry_id/1_jfb7mdpn/version/100001', + 'view_count': int, + 'timestamp': 1533154489, + 'upload_date': '20180801', + 'uploader_id': 'djoyner3', + } + }, { + 'info_dict': { + 'id': '1_8xflxdp7', + 'ext': 'mp4', + 'title': 'CS7646_00-00 Introductio_Textbooks', + 'duration': 37, + 'thumbnail': 'http://cfvod.kaltura.com/p/2019031/sp/201903100/thumbnail/entry_id/1_8xflxdp7/version/100001', + 'view_count': int, + 'timestamp': 1533154512, + 'upload_date': '20180801', + 'uploader_id': 'djoyner3', + } + }, { + 'info_dict': { + 'id': '1_3hqew8kn', + 'ext': 'mp4', + 'title': 'CS7646_00-00 Introductio_Prerequisites', + 'duration': 49, + 'thumbnail': 'http://cfvod.kaltura.com/p/2019031/sp/201903100/thumbnail/entry_id/1_3hqew8kn/version/100001', + 'view_count': int, + 'timestamp': 1533154536, + 'upload_date': '20180801', + 'uploader_id': 'djoyner3', + } + } + ] } ] - @staticmethod - def _extract_url(webpage): - urls = KalturaIE._extract_urls(webpage) - return urls[0] if urls else None - - @staticmethod - def _extract_urls(webpage): + @classmethod + def _extract_embed_urls(cls, url, webpage): # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site finditer = ( list(re.finditer( @@ -160,63 +230,80 @@ class KalturaIE(InfoExtractor): for k, v in embed_info.items(): if v: embed_info[k] = v.strip() - url = 'kaltura:%(partner_id)s:%(id)s' % embed_info + embed_url = 'kaltura:%(partner_id)s:%(id)s' % embed_info escaped_pid = re.escape(embed_info['partner_id']) service_mobj = re.search( r'<script[^>]+src=(["\'])(?P<id>(?:https?:)?//(?:(?!\1).)+)/p/%s/sp/%s00/embedIframeJs' % (escaped_pid, escaped_pid), webpage) if service_mobj: - url = smuggle_url(url, {'service_url': service_mobj.group('id')}) - urls.append(url) + embed_url = smuggle_url(embed_url, {'service_url': service_mobj.group('id')}) + urls.append(embed_url) return urls def _kaltura_api_call(self, video_id, actions, service_url=None, *args, **kwargs): params = actions[0] - if len(actions) > 1: - for i, a in enumerate(actions[1:], start=1): - for k, v in a.items(): - params['%d:%s' % (i, k)] = v + params.update({i: a for i, a in enumerate(actions[1:], start=1)}) data = self._download_json( (service_url or self._SERVICE_URL) + self._SERVICE_BASE, - video_id, query=params, *args, **kwargs) + video_id, data=json.dumps(params).encode('utf-8'), + headers={ + 'Content-Type': 'application/json', + 'Accept-Encoding': 'gzip, deflate, br', + }, *args, **kwargs) - status = data if len(actions) == 1 else data[0] - if status.get('objectType') == 'KalturaAPIException': - raise ExtractorError( - '%s said: %s' % (self.IE_NAME, status['message'])) + for idx, status in enumerate(data): + if not isinstance(status, dict): + continue + if status.get('objectType') == 'KalturaAPIException': + raise ExtractorError( + '%s said: %s (%d)' % (self.IE_NAME, status['message'], idx)) + + data[1] = traverse_obj(data, (1, 'objects', 0)) return data - def _get_video_info(self, video_id, partner_id, service_url=None): + def _get_video_info(self, video_id, partner_id, service_url=None, player_type='html5'): + assert player_type in ('html5', 'kwidget') + if player_type == 'kwidget': + return self._get_video_info_kwidget(video_id, partner_id, service_url) + + return self._get_video_info_html5(video_id, partner_id, service_url) + + def _get_video_info_html5(self, video_id, partner_id, service_url=None): actions = [ { - 'action': 'null', - 'apiVersion': '3.1.5', - 'clientTag': 'kdp:v3.8.5', + 'apiVersion': '3.3.0', + 'clientTag': 'html5:v3.1.0', 'format': 1, # JSON, 2 = XML, 3 = PHP - 'service': 'multirequest', + 'ks': '', + 'partnerId': partner_id, }, { 'expiry': 86400, 'service': 'session', 'action': 'startWidgetSession', - 'widgetId': '_%s' % partner_id, + 'widgetId': self._build_widget_id(partner_id), }, + # info { - 'action': 'get', - 'entryId': video_id, + 'action': 'list', + 'filter': {'redirectFromEntryId': video_id}, 'service': 'baseentry', 'ks': '{1:result:ks}', - 'responseProfile:fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId', - 'responseProfile:type': 1, + 'responseProfile': { + 'type': 1, + 'fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId', + }, }, + # flavor_assets { 'action': 'getbyentryid', 'entryId': video_id, 'service': 'flavorAsset', 'ks': '{1:result:ks}', }, + # captions { 'action': 'list', 'filter:entryIdEqual': video_id, @@ -225,17 +312,85 @@ class KalturaIE(InfoExtractor): }, ] return self._kaltura_api_call( - video_id, actions, service_url, note='Downloading video info JSON') + video_id, actions, service_url, note='Downloading video info JSON (Kaltura html5 player)') + + def _get_video_info_kwidget(self, video_id, partner_id, service_url=None): + actions = [ + { + 'service': 'multirequest', + 'apiVersion': '3.1', + 'expiry': 86400, + 'clientTag': 'kwidget:v2.89', + 'format': 1, # JSON, 2 = XML, 3 = PHP + 'ignoreNull': 1, + 'action': 'null', + }, + # header + { + 'expiry': 86400, + 'service': 'session', + 'action': 'startWidgetSession', + 'widgetId': self._build_widget_id(partner_id), + }, + # (empty) + { + 'expiry': 86400, + 'service': 'session', + 'action': 'startwidgetsession', + 'widgetId': self._build_widget_id(partner_id), + 'format': 9, + 'apiVersion': '3.1', + 'clientTag': 'kwidget:v2.89', + 'ignoreNull': 1, + 'ks': '{1:result:ks}' + }, + # info + { + 'action': 'list', + 'filter': {'redirectFromEntryId': video_id}, + 'service': 'baseentry', + 'ks': '{1:result:ks}', + 'responseProfile': { + 'type': 1, + 'fields': 'createdAt,dataUrl,duration,name,plays,thumbnailUrl,userId', + }, + }, + # flavor_assets + { + 'action': 'getbyentryid', + 'entryId': video_id, + 'service': 'flavorAsset', + 'ks': '{1:result:ks}', + }, + # captions + { + 'action': 'list', + 'filter:entryIdEqual': video_id, + 'service': 'caption_captionasset', + 'ks': '{1:result:ks}', + }, + ] + # second object (representing the second start widget session) is None + header, _, _info, flavor_assets, captions = self._kaltura_api_call( + video_id, actions, service_url, note='Downloading video info JSON (Kaltura kwidget player)') + info = _info['objects'][0] + return header, info, flavor_assets, captions + + def _build_widget_id(self, partner_id): + return partner_id if '_' in partner_id else f'_{partner_id}' + + IFRAME_PACKAGE_DATA_REGEX = r'window\.kalturaIframePackageData\s*=' def _real_extract(self, url): url, smuggled_data = unsmuggle_url(url, {}) mobj = self._match_valid_url(url) - partner_id, entry_id = mobj.group('partner_id', 'id') - ks = None - captions = None + partner_id, entry_id, player_type = mobj.group('partner_id', 'id', 'player_type') + ks, captions = None, None + if not player_type: + player_type = 'kwidget' if 'html5lib/v2' in url else 'html5' if partner_id and entry_id: - _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url')) + _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, smuggled_data.get('service_url'), player_type=player_type) else: path, query = mobj.group('path', 'query') if not path and not query: @@ -247,7 +402,7 @@ class KalturaIE(InfoExtractor): splitted_path = path.split('/') params.update(dict((zip(splitted_path[::2], [[v] for v in splitted_path[1::2]])))) if 'wid' in params: - partner_id = params['wid'][0][1:] + partner_id = remove_start(params['wid'][0], '_') elif 'p' in params: partner_id = params['p'][0] elif 'partner_id' in params: @@ -256,14 +411,13 @@ class KalturaIE(InfoExtractor): raise ExtractorError('Invalid URL', expected=True) if 'entry_id' in params: entry_id = params['entry_id'][0] - _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id) + _, info, flavor_assets, captions = self._get_video_info(entry_id, partner_id, player_type=player_type) elif 'uiconf_id' in params and 'flashvars[referenceId]' in params: reference_id = params['flashvars[referenceId]'][0] webpage = self._download_webpage(url, reference_id) - entry_data = self._parse_json(self._search_regex( - r'window\.kalturaIframePackageData\s*=\s*({.*});', - webpage, 'kalturaIframePackageData'), - reference_id)['entryResult'] + entry_data = self._search_json( + self.IFRAME_PACKAGE_DATA_REGEX, webpage, + 'kalturaIframePackageData', reference_id)['entryResult'] info, flavor_assets = entry_data['meta'], entry_data['contextData']['flavorAssets'] entry_id = info['id'] # Unfortunately, data returned in kalturaIframePackageData lacks @@ -271,16 +425,29 @@ class KalturaIE(InfoExtractor): # regular approach since we now know the entry_id try: _, info, flavor_assets, captions = self._get_video_info( - entry_id, partner_id) + entry_id, partner_id, player_type=player_type) except ExtractorError: # Regular scenario failed but we already have everything # extracted apart from captions and can process at least # with this pass + elif 'uiconf_id' in params and 'flashvars[playlistAPI.kpl0Id]' in params: + playlist_id = params['flashvars[playlistAPI.kpl0Id]'][0] + webpage = self._download_webpage(url, playlist_id) + playlist_data = self._search_json( + self.IFRAME_PACKAGE_DATA_REGEX, webpage, + 'kalturaIframePackageData', playlist_id)['playlistResult'] + return self.playlist_from_matches( + traverse_obj(playlist_data, (playlist_id, 'items', ..., 'id')), + playlist_id, traverse_obj(playlist_data, (playlist_id, 'name')), + ie=KalturaIE, getter=lambda x: f'kaltura:{partner_id}:{x}:{player_type}') else: raise ExtractorError('Invalid URL', expected=True) ks = params.get('flashvars[ks]', [None])[0] + return self._per_video_extract(smuggled_data, entry_id, info, ks, flavor_assets, captions) + + def _per_video_extract(self, smuggled_data, entry_id, info, ks, flavor_assets, captions): source_url = smuggled_data.get('source_url') if source_url: referrer = base64.b64encode( @@ -351,8 +518,6 @@ class KalturaIE(InfoExtractor): formats.extend(fmts) self._merge_subtitles(subs, target=subtitles) - self._sort_formats(formats) - if captions: for caption in captions.get('objects', []): # Continue if caption is not ready @@ -376,5 +541,5 @@ class KalturaIE(InfoExtractor): 'duration': info.get('duration'), 'timestamp': info.get('createdAt'), 'uploader_id': format_field(info, 'userId', ignore=('None', None)), - 'view_count': info.get('plays'), + 'view_count': int_or_none(info.get('plays')), } |