diff options
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 3 | ||||
| -rw-r--r-- | yt_dlp/__init__.py | 23 | ||||
| -rw-r--r-- | yt_dlp/downloader/common.py | 18 | ||||
| -rw-r--r-- | yt_dlp/downloader/external.py | 1 | ||||
| -rw-r--r-- | yt_dlp/downloader/fragment.py | 5 | ||||
| -rw-r--r-- | yt_dlp/options.py | 12 |
6 files changed, 56 insertions, 6 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index ba08f6a7d..749cf9402 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -453,6 +453,9 @@ class YoutubeDL: Allowed keys are 'download', 'postprocess', 'download-title' (console title) and 'postprocess-title'. The template is mapped on a dictionary with keys 'progress' and 'info' + retry_sleep_functions: Dictionary of functions that takes the number of attempts + as argument and returns the time to sleep in seconds. + Allowed keys are 'http', 'fragment', 'file_access' The following parameters are not used by YoutubeDL itself, they are used by the downloader (see yt_dlp/downloader/common.py): diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 81b1716df..b2429f5af 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -247,6 +247,28 @@ def validate_options(opts): opts.extractor_retries = parse_retries('extractor', opts.extractor_retries) opts.file_access_retries = parse_retries('file access', opts.file_access_retries) + # Retry sleep function + def parse_sleep_func(expr): + NUMBER_RE = r'\d+(?:\.\d+)?' + op, start, limit, step, *_ = tuple(re.fullmatch( + rf'(?:(linear|exp)=)?({NUMBER_RE})(?::({NUMBER_RE}))?(?::({NUMBER_RE}))?', + expr.strip()).groups()) + (None, None) + + if op == 'exp': + return lambda n: min(float(start) * (float(step or 2) ** n), float(limit or 'inf')) + else: + default_step = start if op or limit else 0 + return lambda n: min(float(start) + float(step or default_step) * n, float(limit or 'inf')) + + for key, expr in opts.retry_sleep.items(): + if not expr: + del opts.retry_sleep[key] + continue + try: + opts.retry_sleep[key] = parse_sleep_func(expr) + except AttributeError as e: + raise ValueError(f'invalid {key} retry sleep expression {expr!r}: {e}') + # Bytes def parse_bytes(name, value): if value is None: @@ -694,6 +716,7 @@ def parse_options(argv=None): 'file_access_retries': opts.file_access_retries, 'fragment_retries': opts.fragment_retries, 'extractor_retries': opts.extractor_retries, + 'retry_sleep_functions': opts.retry_sleep, 'skip_unavailable_fragments': opts.skip_unavailable_fragments, 'keep_fragments': opts.keep_fragments, 'concurrent_fragment_downloads': opts.concurrent_fragment_downloads, diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 465b5ef99..0b3383071 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -19,6 +19,7 @@ from ..utils import ( encodeFilename, error_to_compat_str, format_bytes, + int_or_none, sanitize_open, shell_quote, timeconvert, @@ -64,6 +65,7 @@ class FileDownloader: useful for bypassing bandwidth throttling imposed by a webserver (experimental) progress_template: See YoutubeDL.py + retry_sleep_functions: See YoutubeDL.py Subclasses of this one must re-define the real_download method. """ @@ -98,6 +100,8 @@ class FileDownloader: def to_screen(self, *args, **kargs): self.ydl.to_screen(*args, quiet=self.params.get('quiet'), **kargs) + __to_screen = to_screen + @property def FD_NAME(self): return re.sub(r'(?<!^)(?=[A-Z])', '_', type(self).__name__[:-2]).lower() @@ -232,7 +236,8 @@ class FileDownloader: self.to_screen( f'[download] Unable to {action} file due to file access error. ' f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...') - time.sleep(0.01) + if not self.sleep_retry('file_access', retry): + time.sleep(0.01) return inner return outer @@ -390,14 +395,23 @@ class FileDownloader: def report_retry(self, err, count, retries): """Report retry in case of HTTP error 5xx""" - self.to_screen( + self.__to_screen( '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...' % (error_to_compat_str(err), count, self.format_retries(retries))) + self.sleep_retry('http', count) def report_unable_to_resume(self): """Report it was impossible to resume download.""" self.to_screen('[download] Unable to resume') + def sleep_retry(self, retry_type, count): + sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type) + delay = int_or_none(sleep_func(n=count - 1)) if sleep_func else None + if delay: + self.__to_screen(f'Sleeping {delay} seconds ...') + time.sleep(delay) + return sleep_func is not None + @staticmethod def supports_manifest(manifest): """ Whether the downloader can download the fragments from the manifest. diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 85c6a6977..812eb45b4 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -142,6 +142,7 @@ class ExternalFD(FragmentFD): self.to_screen( '[%s] Got error. Retrying fragments (attempt %d of %s)...' % (self.get_basename(), count, self.format_retries(fragment_retries))) + self.sleep_retry('fragment', count) if count > fragment_retries: if not skip_unavailable_fragments: self.report_error('Giving up after %s fragment retries' % fragment_retries) diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 4655f067f..410c8c1a4 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -25,10 +25,6 @@ class HttpQuietDownloader(HttpFD): console_title = to_screen - def report_retry(self, err, count, retries): - super().to_screen( - f'[download] Got server HTTP error: {err}. Retrying (attempt {count} of {self.format_retries(retries)}) ...') - class FragmentFD(FileDownloader): """ @@ -70,6 +66,7 @@ class FragmentFD(FileDownloader): self.to_screen( '\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...' % (error_to_compat_str(err), frag_index, count, self.format_retries(retries))) + self.sleep_retry('fragment', count) def report_skip_fragment(self, frag_index, err=None): err = f' {err};' if err else '' diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 1efdc8957..5c97facb7 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -829,6 +829,18 @@ def create_parser(): dest='fragment_retries', metavar='RETRIES', default=10, help='Number of retries for a fragment (default is %default), or "infinite" (DASH, hlsnative and ISM)') downloader.add_option( + '--retry-sleep', + dest='retry_sleep', metavar='[TYPE:]EXPR', default={}, type='str', + action='callback', callback=_dict_from_options_callback, + callback_kwargs={ + 'allowed_keys': 'http|fragment|file_access', + 'default_key': 'http', + }, help=( + 'An expression for the time to sleep between retries in seconds (optionally) prefixed ' + 'by the type of retry (http (default), fragment, file_access) to apply the sleep to. ' + 'EXPR can be a number, or of the forms linear=START[:END[:STEP=1]] or exp=START[:END[:BASE=2]]. ' + 'Eg: --retry-sleep linear=1::2 --retry-sleep fragment:exp=1:20')) + downloader.add_option( '--skip-unavailable-fragments', '--no-abort-on-unavailable-fragment', action='store_true', dest='skip_unavailable_fragments', default=True, help='Skip unavailable fragments for DASH, hlsnative and ISM (default) (Alias: --no-abort-on-unavailable-fragment)') |