From 23326151c45b632c3d5948bd018e80abb370e676 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 19 May 2022 20:00:31 +0530 Subject: Add option --retry-sleep (#3059) Closes #2852 --- yt_dlp/downloader/external.py | 1 + 1 file changed, 1 insertion(+) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 85c6a6977..812eb45b4 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -142,6 +142,7 @@ class ExternalFD(FragmentFD): self.to_screen( '[%s] Got error. Retrying fragments (attempt %d of %s)...' % (self.get_basename(), count, self.format_retries(fragment_retries))) + self.sleep_retry('fragment', count) if count > fragment_retries: if not skip_unavailable_fragments: self.report_error('Giving up after %s fragment retries' % fragment_retries) -- cgit v1.2.3 From 2762dbb17e8556140f9fff0c0aa3373c521f5e09 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 20 May 2022 20:55:21 +0530 Subject: [compat] Add `functools.cached_property` --- yt_dlp/downloader/external.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 812eb45b4..a9da96670 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -5,6 +5,7 @@ import sys import time from .fragment import FragmentFD +from ..compat import functools from ..compat import compat_setenv, compat_str from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( @@ -74,7 +75,7 @@ class ExternalFD(FragmentFD): def EXE_NAME(cls): return cls.get_basename() - @property + @functools.cached_property def exe(self): return self.EXE_NAME -- cgit v1.2.3 From c487cf00101525ff836d59a2a42ef63e85ea9556 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 17 Apr 2022 22:48:50 +0530 Subject: [cleanup] Misc --- yt_dlp/downloader/external.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index a9da96670..66eced1b3 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -1,3 +1,4 @@ +import enum import os.path import re import subprocess @@ -5,8 +6,8 @@ import sys import time from .fragment import FragmentFD -from ..compat import functools -from ..compat import compat_setenv, compat_str +from ..compat import functools # isort: split +from ..compat import compat_setenv from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( Popen, @@ -25,9 +26,14 @@ from ..utils import ( ) +class Features(enum.Enum): + TO_STDOUT = enum.auto() + MULTIPLE_FORMATS = enum.auto() + + class ExternalFD(FragmentFD): SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps') - can_download_to_stdout = False + SUPPORTED_FEATURES = () def real_download(self, filename, info_dict): self.report_destination(filename) @@ -91,9 +97,11 @@ class ExternalFD(FragmentFD): @classmethod def supports(cls, info_dict): - return ( - (cls.can_download_to_stdout or not info_dict.get('to_stdout')) - and info_dict['protocol'] in cls.SUPPORTED_PROTOCOLS) + return all(( + not info_dict.get('to_stdout') or Features.TO_STDOUT in cls.SUPPORTED_FEATURES, + '+' not in info_dict['protocol'] or Features.MULTIPLE_FORMATS in cls.SUPPORTED_FEATURES, + all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+')), + )) @classmethod def can_download(cls, info_dict, path=None): @@ -324,7 +332,7 @@ class HttpieFD(ExternalFD): class FFmpegFD(ExternalFD): SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps', 'm3u8', 'm3u8_native', 'rtsp', 'rtmp', 'rtmp_ffmpeg', 'mms', 'http_dash_segments') - can_download_to_stdout = True + SUPPORTED_FEATURES = (Features.TO_STDOUT, Features.MULTIPLE_FORMATS) @classmethod def available(cls, path=None): @@ -332,10 +340,6 @@ class FFmpegFD(ExternalFD): # Fixme: This may be wrong when --ffmpeg-location is used return FFmpegPostProcessor().available - @classmethod - def supports(cls, info_dict): - return all(proto in cls.SUPPORTED_PROTOCOLS for proto in info_dict['protocol'].split('+')) - def on_process_started(self, proc, stdin): """ Override this in subclasses """ pass @@ -382,10 +386,10 @@ class FFmpegFD(ExternalFD): # start_time = info_dict.get('start_time') or 0 # if start_time: - # args += ['-ss', compat_str(start_time)] + # args += ['-ss', str(start_time)] # end_time = info_dict.get('end_time') # if end_time: - # args += ['-t', compat_str(end_time - start_time)] + # args += ['-t', str(end_time - start_time)] http_headers = None if info_dict.get('http_headers'): @@ -444,7 +448,7 @@ class FFmpegFD(ExternalFD): if isinstance(conn, list): for entry in conn: args += ['-rtmp_conn', entry] - elif isinstance(conn, compat_str): + elif isinstance(conn, str): args += ['-rtmp_conn', conn] for i, url in enumerate(urls): @@ -462,7 +466,7 @@ class FFmpegFD(ExternalFD): args.extend(['-map', f'{i}:{stream_number}']) if self.params.get('test', False): - args += ['-fs', compat_str(self._TEST_FILE_SIZE)] + args += ['-fs', str(self._TEST_FILE_SIZE)] ext = info_dict['ext'] if protocol in ('m3u8', 'm3u8_native'): -- cgit v1.2.3 From 5ec1b6b71689d2f0cbdcd2b6c4dd861fb2fcf911 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 7 Jun 2022 01:43:50 +0530 Subject: Add option `--download-sections` to download video partially Closes #52, Closes #3932 --- yt_dlp/downloader/external.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 66eced1b3..3ef7fd4dc 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -384,13 +384,6 @@ class FFmpegFD(ExternalFD): # http://trac.ffmpeg.org/ticket/6125#comment:10 args += ['-seekable', '1' if seekable else '0'] - # start_time = info_dict.get('start_time') or 0 - # if start_time: - # args += ['-ss', str(start_time)] - # end_time = info_dict.get('end_time') - # if end_time: - # args += ['-t', str(end_time - start_time)] - http_headers = None if info_dict.get('http_headers'): youtubedl_headers = handle_youtubedl_headers(info_dict['http_headers']) @@ -451,15 +444,21 @@ class FFmpegFD(ExternalFD): elif isinstance(conn, str): args += ['-rtmp_conn', conn] + start_time, end_time = info_dict.get('section_start') or 0, info_dict.get('section_end') + for i, url in enumerate(urls): - # We need to specify headers for each http input stream - # otherwise, it will only be applied to the first. - # https://github.com/yt-dlp/yt-dlp/issues/2696 if http_headers is not None and re.match(r'^https?://', url): args += http_headers + if start_time: + args += ['-ss', str(start_time)] + if end_time: + args += ['-t', str(end_time - start_time)] + args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', url] - args += ['-c', 'copy'] + if not (start_time or end_time) or not self.params.get('force_keyframes_at_cuts'): + args += ['-c', 'copy'] + if info_dict.get('requested_formats') or protocol == 'http_dash_segments': for (i, fmt) in enumerate(info_dict.get('requested_formats') or [info_dict]): stream_number = fmt.get('manifest_stream_number', 0) -- cgit v1.2.3 From f0c9fb96827ff798a48626e7e5d32a9c5de7b97e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 16 Jun 2022 02:25:43 +0530 Subject: [utils] `Popen`: Refactor to use contextmanager Fixes https://github.com/yt-dlp/yt-dlp/issues/3531#issuecomment-1156223597 --- yt_dlp/downloader/external.py | 66 +++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 37 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 3ef7fd4dc..a1cb07e05 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -34,6 +34,7 @@ class Features(enum.Enum): class ExternalFD(FragmentFD): SUPPORTED_PROTOCOLS = ('http', 'https', 'ftp', 'ftps') SUPPORTED_FEATURES = () + _CAPTURE_STDERR = True def real_download(self, filename, info_dict): self.report_destination(filename) @@ -128,24 +129,25 @@ class ExternalFD(FragmentFD): self._debug_cmd(cmd) if 'fragments' not in info_dict: - p = Popen(cmd, stderr=subprocess.PIPE) - _, stderr = p.communicate_or_kill() - if p.returncode != 0: - self.to_stderr(stderr.decode('utf-8', 'replace')) - return p.returncode + _, stderr, returncode = Popen.run( + cmd, text=True, stderr=subprocess.PIPE if self._CAPTURE_STDERR else None) + if returncode and stderr: + self.to_stderr(stderr) + return returncode fragment_retries = self.params.get('fragment_retries', 0) skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True) count = 0 while count <= fragment_retries: - p = Popen(cmd, stderr=subprocess.PIPE) - _, stderr = p.communicate_or_kill() - if p.returncode == 0: + _, stderr, returncode = Popen.run(cmd, text=True, stderr=subprocess.PIPE) + if not returncode: break + # TODO: Decide whether to retry based on error code # https://aria2.github.io/manual/en/html/aria2c.html#exit-status - self.to_stderr(stderr.decode('utf-8', 'replace')) + if stderr: + self.to_stderr(stderr) count += 1 if count <= fragment_retries: self.to_screen( @@ -180,6 +182,7 @@ class ExternalFD(FragmentFD): class CurlFD(ExternalFD): AVAILABLE_OPT = '-V' + _CAPTURE_STDERR = False # curl writes the progress to stderr def _make_cmd(self, tmpfilename, info_dict): cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed'] @@ -204,16 +207,6 @@ class CurlFD(ExternalFD): cmd += ['--', info_dict['url']] return cmd - def _call_downloader(self, tmpfilename, info_dict): - cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)] - - self._debug_cmd(cmd) - - # curl writes the progress to stderr so don't capture it. - p = Popen(cmd) - p.communicate_or_kill() - return p.returncode - class AxelFD(ExternalFD): AVAILABLE_OPT = '-V' @@ -500,24 +493,23 @@ class FFmpegFD(ExternalFD): args.append(encodeFilename(ffpp._ffmpeg_filename_argument(tmpfilename), True)) self._debug_cmd(args) - proc = Popen(args, stdin=subprocess.PIPE, env=env) - if url in ('-', 'pipe:'): - self.on_process_started(proc, proc.stdin) - try: - retval = proc.wait() - except BaseException as e: - # subprocces.run would send the SIGKILL signal to ffmpeg and the - # mp4 file couldn't be played, but if we ask ffmpeg to quit it - # produces a file that is playable (this is mostly useful for live - # streams). Note that Windows is not affected and produces playable - # files (see https://github.com/ytdl-org/youtube-dl/issues/8300). - if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and url not in ('-', 'pipe:'): - proc.communicate_or_kill(b'q') - else: - proc.kill() - proc.wait() - raise - return retval + with Popen(args, stdin=subprocess.PIPE, env=env) as proc: + if url in ('-', 'pipe:'): + self.on_process_started(proc, proc.stdin) + try: + retval = proc.wait() + except BaseException as e: + # subprocces.run would send the SIGKILL signal to ffmpeg and the + # mp4 file couldn't be played, but if we ask ffmpeg to quit it + # produces a file that is playable (this is mostly useful for live + # streams). Note that Windows is not affected and produces playable + # files (see https://github.com/ytdl-org/youtube-dl/issues/8300). + if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32' and url not in ('-', 'pipe:'): + proc.communicate_or_kill(b'q') + else: + proc.kill(timeout=None) + raise + return retval class AVconvFD(FFmpegFD): -- cgit v1.2.3 From ac668111128b5f124b4271b3aa4c35f6e71a4749 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 24 Jun 2022 13:40:17 +0530 Subject: [compat] Remove more functions Removing any more will require changes to a large number of extractors --- yt_dlp/downloader/external.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index a1cb07e05..dee945aff 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -7,7 +7,6 @@ import time from .fragment import FragmentFD from ..compat import functools # isort: split -from ..compat import compat_setenv from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( Popen, @@ -403,8 +402,8 @@ class FFmpegFD(ExternalFD): # We could switch to the following code if we are able to detect version properly # args += ['-http_proxy', proxy] env = os.environ.copy() - compat_setenv('HTTP_PROXY', proxy, env=env) - compat_setenv('http_proxy', proxy, env=env) + env['HTTP_PROXY'] = proxy + env['http_proxy'] = proxy protocol = info_dict.get('protocol') -- cgit v1.2.3 From 14f25df2b6233553e968df023430ca96c0b1df9f Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 24 Jun 2022 16:24:43 +0530 Subject: [compat] Remove deprecated functions from core code --- yt_dlp/downloader/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index dee945aff..f84a17f23 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -6,7 +6,7 @@ import sys import time from .fragment import FragmentFD -from ..compat import functools # isort: split +from ..compat import functools from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( Popen, -- cgit v1.2.3