diff options
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 63 | ||||
| -rw-r--r-- | yt_dlp/downloader/__init__.py | 17 | ||||
| -rw-r--r-- | yt_dlp/downloader/dash.py | 10 | ||||
| -rw-r--r-- | yt_dlp/downloader/external.py | 10 | ||||
| -rw-r--r-- | yt_dlp/downloader/hls.py | 7 | ||||
| -rw-r--r-- | yt_dlp/downloader/niconico.py | 4 | ||||
| -rw-r--r-- | yt_dlp/extractor/common.py | 4 | ||||
| -rw-r--r-- | yt_dlp/options.py | 17 | ||||
| -rw-r--r-- | yt_dlp/utils.py | 4 |
9 files changed, 67 insertions, 69 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 5b1ee8ee4..aa8a54a55 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -126,6 +126,7 @@ from .extractor import ( ) from .extractor.openload import PhantomJSwrapper from .downloader import ( + FFmpegFD, get_suitable_downloader, shorten_protocol_name ) @@ -2690,20 +2691,15 @@ class YoutubeDL(object): info_dict['__real_download'] = False _protocols = set(determine_protocol(f) for f in requested_formats) - if len(_protocols) == 1: + if len(_protocols) == 1: # All requested formats have same protocol info_dict['protocol'] = _protocols.pop() - directly_mergable = ( - 'no-direct-merge' not in self.params.get('compat_opts', []) - and info_dict.get('protocol') is not None # All requested formats have same protocol - and not self.params.get('allow_unplayable_formats') - and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD') - if directly_mergable: - info_dict['url'] = requested_formats[0]['url'] - # Treat it as a single download - dl_filename = existing_file(full_filename, temp_filename) - if dl_filename is None: - success, real_download = self.dl(temp_filename, info_dict) - info_dict['__real_download'] = real_download + directly_mergable = FFmpegFD.can_merge_formats(info_dict) + if dl_filename is not None: + pass + elif (directly_mergable and get_suitable_downloader(info_dict, self.params) == FFmpegFD): + info_dict['url'] = '\n'.join(f['url'] for f in requested_formats) + success, real_download = self.dl(temp_filename, info_dict) + info_dict['__real_download'] = real_download else: downloaded = [] merger = FFmpegMergerPP(self) @@ -2717,28 +2713,25 @@ class YoutubeDL(object): 'You have requested merging of multiple formats but ffmpeg is not installed. ' 'The formats won\'t be merged.') - if dl_filename is None: - for f in requested_formats: - new_info = dict(info_dict) - del new_info['requested_formats'] - new_info.update(f) - fname = prepend_extension( - self.prepare_filename(new_info, 'temp'), - 'f%s' % f['format_id'], new_info['ext']) - if not self._ensure_dir_exists(fname): - return - downloaded.append(fname) - partial_success, real_download = self.dl(fname, new_info) - info_dict['__real_download'] = info_dict['__real_download'] or real_download - success = success and partial_success - if merger.available and not self.params.get('allow_unplayable_formats'): - info_dict['__postprocessors'].append(merger) - info_dict['__files_to_merge'] = downloaded - # Even if there were no downloads, it is being merged only now - info_dict['__real_download'] = True - else: - for file in downloaded: - files_to_move[file] = None + for f in requested_formats: + new_info = dict(info_dict) + del new_info['requested_formats'] + new_info.update(f) + fname = prepend_extension(temp_filename, 'f%s' % f['format_id'], new_info['ext']) + if not self._ensure_dir_exists(fname): + return + downloaded.append(fname) + partial_success, real_download = self.dl(fname, new_info) + info_dict['__real_download'] = info_dict['__real_download'] or real_download + success = success and partial_success + if merger.available and not self.params.get('allow_unplayable_formats'): + info_dict['__postprocessors'].append(merger) + info_dict['__files_to_merge'] = downloaded + # Even if there were no downloads, it is being merged only now + info_dict['__real_download'] = True + else: + for file in downloaded: + files_to_move[file] = None else: # Just a single file dl_filename = existing_file(full_filename, temp_filename) diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py index 6769cf8e6..53393e89f 100644 --- a/yt_dlp/downloader/__init__.py +++ b/yt_dlp/downloader/__init__.py @@ -3,17 +3,19 @@ from __future__ import unicode_literals from ..compat import compat_str from ..utils import ( determine_protocol, + NO_DEFAULT ) -def _get_real_downloader(info_dict, protocol=None, *args, **kwargs): +def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None): + info_dict['protocol'] = determine_protocol(info_dict) info_copy = info_dict.copy() if protocol: info_copy['protocol'] = protocol - return get_suitable_downloader(info_copy, *args, **kwargs) + return _get_suitable_downloader(info_copy, params, default) -# Some of these require _get_real_downloader +# Some of these require get_suitable_downloader from .common import FileDownloader from .dash import DashSegmentsFD from .f4m import F4mFD @@ -69,14 +71,15 @@ def shorten_protocol_name(proto, simplify=False): return short_protocol_names.get(proto, proto) -def get_suitable_downloader(info_dict, params={}, default=HttpFD): +def _get_suitable_downloader(info_dict, params, default): """Get the downloader class that can handle the info dict.""" - protocol = determine_protocol(info_dict) - info_dict['protocol'] = protocol + if default is NO_DEFAULT: + default = HttpFD # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict): # return FFmpegFD + protocol = info_dict['protocol'] downloaders = params.get('external_downloader') external_downloader = ( downloaders if isinstance(downloaders, compat_str) or downloaders is None @@ -94,7 +97,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD): return FFmpegFD elif external_downloader == 'native': return HlsFD - elif _get_real_downloader(info_dict, 'm3u8_frag_urls', params, None): + elif get_suitable_downloader(info_dict, params, None, protocol='m3u8_frag_urls'): return HlsFD elif params.get('hls_prefer_native') is True: return HlsFD diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py index 9dae6b9bd..ccc41e158 100644 --- a/yt_dlp/downloader/dash.py +++ b/yt_dlp/downloader/dash.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from ..downloader import _get_real_downloader +from ..downloader import get_suitable_downloader from .fragment import FragmentFD from ..utils import urljoin @@ -15,11 +15,14 @@ class DashSegmentsFD(FragmentFD): FD_NAME = 'dashsegments' def real_download(self, filename, info_dict): + if info_dict.get('is_live'): + self.report_error('Live DASH videos are not supported') + fragment_base_url = info_dict.get('fragment_base_url') fragments = info_dict['fragments'][:1] if self.params.get( 'test', False) else info_dict['fragments'] - real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None) + real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='dash_frag_urls') ctx = { 'filename': filename, @@ -54,9 +57,6 @@ class DashSegmentsFD(FragmentFD): info_copy = info_dict.copy() info_copy['fragments'] = fragments_to_download fd = real_downloader(self.ydl, self.params) - # TODO: Make progress updates work without hooking twice - # for ph in self._progress_hooks: - # fd.add_progress_hook(ph) return fd.real_download(filename, info_copy) return self.download_and_append_fragments(ctx, fragments_to_download, info_dict) diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index d0ee745b3..b2a605458 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -345,12 +345,22 @@ class FFmpegFD(ExternalFD): @classmethod def available(cls, path=None): # TODO: Fix path for ffmpeg + # Fixme: This may be wrong when --ffmpeg-location is used return FFmpegPostProcessor().available def on_process_started(self, proc, stdin): """ Override this in subclasses """ pass + @classmethod + def can_merge_formats(cls, info_dict, params={}): + return ( + info_dict.get('requested_formats') + and info_dict.get('protocol') + and not params.get('allow_unplayable_formats') + and 'no-direct-merge' not in params.get('compat_opts', []) + and cls.can_download(info_dict)) + def _call_downloader(self, tmpfilename, info_dict): urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']] ffpp = FFmpegPostProcessor(downloader=self) diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index 64637badf..79d4ad5e9 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -4,7 +4,7 @@ import re import io import binascii -from ..downloader import _get_real_downloader +from ..downloader import get_suitable_downloader from .fragment import FragmentFD, can_decrypt_frag from .external import FFmpegFD @@ -80,16 +80,13 @@ class HlsFD(FragmentFD): fd = FFmpegFD(self.ydl, self.params) self.report_warning( '%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename())) - # TODO: Make progress updates work without hooking twice - # for ph in self._progress_hooks: - # fd.add_progress_hook(ph) return fd.real_download(filename, info_dict) is_webvtt = info_dict['ext'] == 'vtt' if is_webvtt: real_downloader = None # Packing the fragments is not currently supported for external downloader else: - real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None) + real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='m3u8_frag_urls') if real_downloader and not real_downloader.supports_manifest(s): real_downloader = None if real_downloader: diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py index c5a3587a4..256840d68 100644 --- a/yt_dlp/downloader/niconico.py +++ b/yt_dlp/downloader/niconico.py @@ -4,7 +4,7 @@ from __future__ import unicode_literals import threading from .common import FileDownloader -from ..downloader import _get_real_downloader +from ..downloader import get_suitable_downloader from ..extractor.niconico import NiconicoIE from ..compat import compat_urllib_request @@ -20,7 +20,7 @@ class NiconicoDmcFD(FileDownloader): ie = NiconicoIE(self.ydl) info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict) - fd = _get_real_downloader(info_dict, params=self.params)(self.ydl, self.params) + fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params) success = download_complete = False timer = [None] diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index e767d75b5..a3ac9dfb7 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1298,7 +1298,7 @@ class InfoExtractor(object): # JSON-LD may be malformed and thus `fatal` should be respected. # At the same time `default` may be passed that assumes `fatal=False` # for _search_regex. Let's simulate the same behavior here as well. - fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False + fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False json_ld = [] for mobj in json_ld_list: json_ld_item = self._parse_json( @@ -1522,7 +1522,7 @@ class InfoExtractor(object): 'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')}, 'ext': {'type': 'combined', 'field': ('vext', 'aext')}, 'res': {'type': 'multiple', 'field': ('height', 'width'), - 'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))}, + 'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))}, # Most of these exist only for compatibility reasons 'dimension': {'type': 'alias', 'field': 'res'}, diff --git a/yt_dlp/options.py b/yt_dlp/options.py index e21030c28..85dadc7d9 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -706,9 +706,8 @@ def parseOpts(overrideArguments=None): callback_kwargs={ 'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms', 'default_key': 'default', - 'process': lambda x: x.strip() - }, - help=( + 'process': str.strip + }, help=( 'Name or path of the external downloader to use (optionally) prefixed by ' 'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. ' 'Currently supports native, %s (Recommended: aria2c). ' @@ -724,8 +723,7 @@ def parseOpts(overrideArguments=None): 'allowed_keys': '|'.join(list_external_downloaders()), 'default_key': 'default', 'process': compat_shlex_split - }, - help=( + }, help=( 'Give these arguments to the external downloader. ' 'Specify the downloader name and the arguments separated by a colon ":". ' 'You can use this option multiple times to give different arguments to different downloaders ' @@ -944,8 +942,7 @@ def parseOpts(overrideArguments=None): callback_kwargs={ 'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()), 'default_key': 'default' - }, - help='Output filename template; see "OUTPUT TEMPLATE" for details') + }, help='Output filename template; see "OUTPUT TEMPLATE" for details') filesystem.add_option( '--output-na-placeholder', dest='outtmpl_na_placeholder', metavar='TEXT', default='NA', @@ -1191,8 +1188,7 @@ def parseOpts(overrideArguments=None): 'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat', 'process': compat_shlex_split, 'multiple_keys': False - }, - help=( + }, help=( 'Give these arguments to the postprocessors. ' 'Specify the postprocessor/executable name and the arguments separated by a colon ":" ' 'to give the argument to the specified postprocessor/executable. Supported PP are: ' @@ -1385,8 +1381,7 @@ def parseOpts(overrideArguments=None): 'multiple_keys': False, 'process': lambda val: dict( _extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';')) - }, - help=( + }, help=( 'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. ' 'You can use this option multiple times to give arguments for different extractors')) extractor.add_option( diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 4f584596d..f24e00b02 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4544,7 +4544,7 @@ def parse_codecs(codecs_str): if not codecs_str: return {} split_codecs = list(filter(None, map( - lambda str: str.strip(), codecs_str.strip().strip(',').split(',')))) + str.strip, codecs_str.strip().strip(',').split(',')))) vcodec, acodec = None, None for full_codec in split_codecs: codec = full_codec.split('.')[0] @@ -6246,7 +6246,7 @@ def traverse_obj( # TODO: Write tests ''' if not casesense: - _lower = lambda k: k.lower() if isinstance(k, str) else k + _lower = lambda k: (k.lower() if isinstance(k, str) else k) path_list = (map(_lower, variadic(path)) for path in path_list) def _traverse_obj(obj, path, _current_depth=0): |