aboutsummaryrefslogtreecommitdiffstats
path: root/yt_dlp/downloader
diff options
context:
space:
mode:
Diffstat (limited to 'yt_dlp/downloader')
-rw-r--r--yt_dlp/downloader/__init__.py8
-rw-r--r--yt_dlp/downloader/common.py104
-rw-r--r--yt_dlp/downloader/dash.py16
-rw-r--r--yt_dlp/downloader/external.py46
-rw-r--r--yt_dlp/downloader/f4m.py6
-rw-r--r--yt_dlp/downloader/fragment.py100
-rw-r--r--yt_dlp/downloader/hls.py2
-rw-r--r--yt_dlp/downloader/http.py40
-rw-r--r--yt_dlp/downloader/ism.py27
-rw-r--r--yt_dlp/downloader/mhtml.py14
-rw-r--r--yt_dlp/downloader/websocket.py2
-rw-r--r--yt_dlp/downloader/youtube_live_chat.py37
12 files changed, 190 insertions, 212 deletions
diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py
index a7dc6c9d0..c34dbcea9 100644
--- a/yt_dlp/downloader/__init__.py
+++ b/yt_dlp/downloader/__init__.py
@@ -59,10 +59,11 @@ PROTOCOL_MAP = {
def shorten_protocol_name(proto, simplify=False):
short_protocol_names = {
- 'm3u8_native': 'm3u8_n',
- 'rtmp_ffmpeg': 'rtmp_f',
+ 'm3u8_native': 'm3u8',
+ 'm3u8': 'm3u8F',
+ 'rtmp_ffmpeg': 'rtmpF',
'http_dash_segments': 'dash',
- 'http_dash_segments_generator': 'dash_g',
+ 'http_dash_segments_generator': 'dashG',
'niconico_dmc': 'dmc',
'websocket_frag': 'WSfrag',
}
@@ -70,6 +71,7 @@ def shorten_protocol_name(proto, simplify=False):
short_protocol_names.update({
'https': 'http',
'ftps': 'ftp',
+ 'm3u8': 'm3u8', # Reverse above m3u8 mapping
'm3u8_native': 'm3u8',
'http_dash_segments_generator': 'dash',
'rtmp_ffmpeg': 'rtmp',
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 3a0a014ef..fe3633250 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -1,5 +1,6 @@
import contextlib
import errno
+import functools
import os
import random
import re
@@ -12,16 +13,18 @@ from ..minicurses import (
QuietMultilinePrinter,
)
from ..utils import (
- NUMBER_RE,
+ IDENTITY,
+ NO_DEFAULT,
LockingUnsupportedError,
Namespace,
+ RetryManager,
classproperty,
decodeArgument,
encodeFilename,
- error_to_compat_str,
- float_or_none,
format_bytes,
join_nonempty,
+ parse_bytes,
+ remove_start,
sanitize_open,
shell_quote,
timeconvert,
@@ -90,6 +93,7 @@ class FileDownloader:
for func in (
'deprecation_warning',
+ 'deprecated_feature',
'report_error',
'report_file_already_downloaded',
'report_warning',
@@ -117,11 +121,11 @@ class FileDownloader:
time = timetuple_from_msec(seconds * 1000)
if time.hours > 99:
return '--:--:--'
- if not time.hours:
- return '%02d:%02d' % time[1:-1]
return '%02d:%02d:%02d' % time[:-1]
- format_eta = format_seconds
+ @classmethod
+ def format_eta(cls, seconds):
+ return f'{remove_start(cls.format_seconds(seconds), "00:"):>8s}'
@staticmethod
def calc_percent(byte_counter, data_len):
@@ -176,12 +180,7 @@ class FileDownloader:
@staticmethod
def parse_bytes(bytestr):
"""Parse a string indicating a byte quantity into an integer."""
- matchobj = re.match(rf'(?i)^({NUMBER_RE})([kMGTPEZY]?)$', bytestr)
- if matchobj is None:
- return None
- number = float(matchobj.group(1))
- multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
- return int(round(number * multiplier))
+ parse_bytes(bytestr)
def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit."""
@@ -215,27 +214,24 @@ class FileDownloader:
return filename + '.ytdl'
def wrap_file_access(action, *, fatal=False):
- def outer(func):
- def inner(self, *args, **kwargs):
- file_access_retries = self.params.get('file_access_retries', 0)
- retry = 0
- while True:
- try:
- return func(self, *args, **kwargs)
- except OSError as err:
- retry = retry + 1
- if retry > file_access_retries or err.errno not in (errno.EACCES, errno.EINVAL):
- if not fatal:
- self.report_error(f'unable to {action} file: {err}')
- return
- raise
- self.to_screen(
- f'[download] Unable to {action} file due to file access error. '
- f'Retrying (attempt {retry} of {self.format_retries(file_access_retries)}) ...')
- if not self.sleep_retry('file_access', retry):
- time.sleep(0.01)
- return inner
- return outer
+ def error_callback(err, count, retries, *, fd):
+ return RetryManager.report_retry(
+ err, count, retries, info=fd.__to_screen,
+ warn=lambda e: (time.sleep(0.01), fd.to_screen(f'[download] Unable to {action} file: {e}')),
+ error=None if fatal else lambda e: fd.report_error(f'Unable to {action} file: {e}'),
+ sleep_func=fd.params.get('retry_sleep_functions', {}).get('file_access'))
+
+ def wrapper(self, func, *args, **kwargs):
+ for retry in RetryManager(self.params.get('file_access_retries'), error_callback, fd=self):
+ try:
+ return func(self, *args, **kwargs)
+ except OSError as err:
+ if err.errno in (errno.EACCES, errno.EINVAL):
+ retry.error = err
+ continue
+ retry.error_callback(err, 1, 0)
+
+ return functools.partial(functools.partialmethod, wrapper)
@wrap_file_access('open', fatal=True)
def sanitize_open(self, filename, open_mode):
@@ -332,11 +328,16 @@ class FileDownloader:
return tmpl
return default
+ _format_bytes = lambda k: f'{format_bytes(s.get(k)):>10s}'
+
if s['status'] == 'finished':
if self.params.get('noprogress'):
self.to_screen('[download] Download completed')
+ speed = try_call(lambda: s['total_bytes'] / s['elapsed'])
s.update({
- '_total_bytes_str': format_bytes(s.get('total_bytes')),
+ 'speed': speed,
+ '_speed_str': self.format_speed(speed).strip(),
+ '_total_bytes_str': _format_bytes('total_bytes'),
'_elapsed_str': self.format_seconds(s.get('elapsed')),
'_percent_str': self.format_percent(100),
})
@@ -344,21 +345,22 @@ class FileDownloader:
'100%%',
with_fields(('total_bytes', 'of %(_total_bytes_str)s')),
with_fields(('elapsed', 'in %(_elapsed_str)s')),
+ with_fields(('speed', 'at %(_speed_str)s')),
delim=' '))
if s['status'] != 'downloading':
return
s.update({
- '_eta_str': self.format_eta(s.get('eta')),
+ '_eta_str': self.format_eta(s.get('eta')).strip(),
'_speed_str': self.format_speed(s.get('speed')),
'_percent_str': self.format_percent(try_call(
lambda: 100 * s['downloaded_bytes'] / s['total_bytes'],
lambda: 100 * s['downloaded_bytes'] / s['total_bytes_estimate'],
lambda: s['downloaded_bytes'] == 0 and 0)),
- '_total_bytes_str': format_bytes(s.get('total_bytes')),
- '_total_bytes_estimate_str': format_bytes(s.get('total_bytes_estimate')),
- '_downloaded_bytes_str': format_bytes(s.get('downloaded_bytes')),
+ '_total_bytes_str': _format_bytes('total_bytes'),
+ '_total_bytes_estimate_str': _format_bytes('total_bytes_estimate'),
+ '_downloaded_bytes_str': _format_bytes('downloaded_bytes'),
'_elapsed_str': self.format_seconds(s.get('elapsed')),
})
@@ -378,25 +380,20 @@ class FileDownloader:
"""Report attempt to resume at given byte."""
self.to_screen('[download] Resuming download at byte %s' % resume_len)
- def report_retry(self, err, count, retries):
- """Report retry in case of HTTP error 5xx"""
- self.__to_screen(
- '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
- % (error_to_compat_str(err), count, self.format_retries(retries)))
- self.sleep_retry('http', count)
+ def report_retry(self, err, count, retries, frag_index=NO_DEFAULT, fatal=True):
+ """Report retry"""
+ is_frag = False if frag_index is NO_DEFAULT else 'fragment'
+ RetryManager.report_retry(
+ err, count, retries, info=self.__to_screen,
+ warn=lambda msg: self.__to_screen(f'[download] Got error: {msg}'),
+ error=IDENTITY if not fatal else lambda e: self.report_error(f'\r[download] Got error: {e}'),
+ sleep_func=self.params.get('retry_sleep_functions', {}).get(is_frag or 'http'),
+ suffix=f'fragment{"s" if frag_index is None else f" {frag_index}"}' if is_frag else None)
def report_unable_to_resume(self):
"""Report it was impossible to resume download."""
self.to_screen('[download] Unable to resume')
- def sleep_retry(self, retry_type, count):
- sleep_func = self.params.get('retry_sleep_functions', {}).get(retry_type)
- delay = float_or_none(sleep_func(n=count - 1)) if sleep_func else None
- if delay:
- self.__to_screen(f'Sleeping {delay:.2f} seconds ...')
- time.sleep(delay)
- return sleep_func is not None
-
@staticmethod
def supports_manifest(manifest):
""" Whether the downloader can download the fragments from the manifest.
@@ -450,8 +447,7 @@ class FileDownloader:
raise NotImplementedError('This method must be implemented by subclasses')
def _hook_progress(self, status, info_dict):
- if not self._progress_hooks:
- return
+ # Ideally we want to make a copy of the dict, but that is too slow
status['info_dict'] = info_dict
# youtube-dl passes the same status object to all the hooks.
# Some third party scripts seems to be relying on this.
diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py
index a6da26f09..4328d739c 100644
--- a/yt_dlp/downloader/dash.py
+++ b/yt_dlp/downloader/dash.py
@@ -1,8 +1,9 @@
import time
+import urllib.parse
from . import get_suitable_downloader
from .fragment import FragmentFD
-from ..utils import urljoin
+from ..utils import update_url_query, urljoin
class DashSegmentsFD(FragmentFD):
@@ -40,7 +41,12 @@ class DashSegmentsFD(FragmentFD):
self._prepare_and_start_frag_download(ctx, fmt)
ctx['start'] = real_start
- fragments_to_download = self._get_fragments(fmt, ctx)
+ extra_query = None
+ extra_param_to_segment_url = info_dict.get('extra_param_to_segment_url')
+ if extra_param_to_segment_url:
+ extra_query = urllib.parse.parse_qs(extra_param_to_segment_url)
+
+ fragments_to_download = self._get_fragments(fmt, ctx, extra_query)
if real_downloader:
self.to_screen(
@@ -51,13 +57,13 @@ class DashSegmentsFD(FragmentFD):
args.append([ctx, fragments_to_download, fmt])
- return self.download_and_append_fragments_multiple(*args)
+ return self.download_and_append_fragments_multiple(*args, is_fatal=lambda idx: idx == 0)
def _resolve_fragments(self, fragments, ctx):
fragments = fragments(ctx) if callable(fragments) else fragments
return [next(iter(fragments))] if self.params.get('test') else fragments
- def _get_fragments(self, fmt, ctx):
+ def _get_fragments(self, fmt, ctx, extra_query):
fragment_base_url = fmt.get('fragment_base_url')
fragments = self._resolve_fragments(fmt['fragments'], ctx)
@@ -70,6 +76,8 @@ class DashSegmentsFD(FragmentFD):
if not fragment_url:
assert fragment_base_url
fragment_url = urljoin(fragment_base_url, fragment['path'])
+ if extra_query:
+ fragment_url = update_url_query(fragment_url, extra_query)
yield {
'frag_index': frag_index,
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index f84a17f23..895390d6c 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -10,6 +10,7 @@ from ..compat import functools
from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
from ..utils import (
Popen,
+ RetryManager,
_configuration_args,
check_executable,
classproperty,
@@ -134,29 +135,22 @@ class ExternalFD(FragmentFD):
self.to_stderr(stderr)
return returncode
- fragment_retries = self.params.get('fragment_retries', 0)
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
- count = 0
- while count <= fragment_retries:
+ retry_manager = RetryManager(self.params.get('fragment_retries'), self.report_retry,
+ frag_index=None, fatal=not skip_unavailable_fragments)
+ for retry in retry_manager:
_, stderr, returncode = Popen.run(cmd, text=True, stderr=subprocess.PIPE)
if not returncode:
break
-
# TODO: Decide whether to retry based on error code
# https://aria2.github.io/manual/en/html/aria2c.html#exit-status
if stderr:
self.to_stderr(stderr)
- count += 1
- if count <= fragment_retries:
- self.to_screen(
- '[%s] Got error. Retrying fragments (attempt %d of %s)...'
- % (self.get_basename(), count, self.format_retries(fragment_retries)))
- self.sleep_retry('fragment', count)
- if count > fragment_retries:
- if not skip_unavailable_fragments:
- self.report_error('Giving up after %s fragment retries' % fragment_retries)
- return -1
+ retry.error = Exception()
+ continue
+ if not skip_unavailable_fragments and retry_manager.error:
+ return -1
decrypt_fragment = self.decrypter(info_dict)
dest, _ = self.sanitize_open(tmpfilename, 'wb')
@@ -258,6 +252,10 @@ class Aria2cFD(ExternalFD):
check_results = (not re.search(feature, manifest) for feature in UNSUPPORTED_FEATURES)
return all(check_results)
+ @staticmethod
+ def _aria2c_filename(fn):
+ return fn if os.path.isabs(fn) else f'.{os.path.sep}{fn}'
+
def _make_cmd(self, tmpfilename, info_dict):
cmd = [self.exe, '-c',
'--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
@@ -286,11 +284,9 @@ class Aria2cFD(ExternalFD):
# https://github.com/aria2/aria2/issues/1373
dn = os.path.dirname(tmpfilename)
if dn:
- if not os.path.isabs(dn):
- dn = f'.{os.path.sep}{dn}'
- cmd += ['--dir', dn + os.path.sep]
+ cmd += ['--dir', self._aria2c_filename(dn) + os.path.sep]
if 'fragments' not in info_dict:
- cmd += ['--out', f'.{os.path.sep}{os.path.basename(tmpfilename)}']
+ cmd += ['--out', self._aria2c_filename(os.path.basename(tmpfilename))]
cmd += ['--auto-file-renaming=false']
if 'fragments' in info_dict:
@@ -299,11 +295,11 @@ class Aria2cFD(ExternalFD):
url_list = []
for frag_index, fragment in enumerate(info_dict['fragments']):
fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
- url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename))
+ url_list.append('%s\n\tout=%s' % (fragment['url'], self._aria2c_filename(fragment_filename)))
stream, _ = self.sanitize_open(url_list_file, 'wb')
stream.write('\n'.join(url_list).encode())
stream.close()
- cmd += ['-i', url_list_file]
+ cmd += ['-i', self._aria2c_filename(url_list_file)]
else:
cmd += ['--', info_dict['url']]
return cmd
@@ -521,16 +517,14 @@ _BY_NAME = {
if name.endswith('FD') and name not in ('ExternalFD', 'FragmentFD')
}
-_BY_EXE = {klass.EXE_NAME: klass for klass in _BY_NAME.values()}
-
def list_external_downloaders():
return sorted(_BY_NAME.keys())
def get_external_downloader(external_downloader):
- """ Given the name of the executable, see whether we support the given
- downloader . """
- # Drop .exe extension on Windows
+ """ Given the name of the executable, see whether we support the given downloader """
bn = os.path.splitext(os.path.basename(external_downloader))[0]
- return _BY_NAME.get(bn, _BY_EXE.get(bn))
+ return _BY_NAME.get(bn) or next((
+ klass for klass in _BY_NAME.values() if klass.EXE_NAME in bn
+ ), None)
diff --git a/yt_dlp/downloader/f4m.py b/yt_dlp/downloader/f4m.py
index 770354de7..306f92192 100644
--- a/yt_dlp/downloader/f4m.py
+++ b/yt_dlp/downloader/f4m.py
@@ -184,7 +184,7 @@ def build_fragments_list(boot_info):
first_frag_number = fragment_run_entry_table[0]['first']
fragments_counter = itertools.count(first_frag_number)
for segment, fragments_count in segment_run_table['segment_run']:
- # In some live HDS streams (for example Rai), `fragments_count` is
+ # In some live HDS streams (e.g. Rai), `fragments_count` is
# abnormal and causing out-of-memory errors. It's OK to change the
# number of fragments for live streams as they are updated periodically
if fragments_count == 4294967295 and boot_info['live']:
@@ -424,6 +424,4 @@ class F4mFD(FragmentFD):
msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
self.report_warning(msg)
- self._finish_frag_download(ctx, info_dict)
-
- return True
+ return self._finish_frag_download(ctx, info_dict)
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 3535e0e7d..83f7870ed 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -14,8 +14,8 @@ from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
from ..compat import compat_os_name
from ..utils import (
DownloadError,
+ RetryManager,
encodeFilename,
- error_to_compat_str,
sanitized_Request,
traverse_obj,
)
@@ -65,10 +65,9 @@ class FragmentFD(FileDownloader):
"""
def report_retry_fragment(self, err, frag_index, count, retries):
- self.to_screen(
- '\r[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s) ...'
- % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
- self.sleep_retry('fragment', count)
+ self.deprecation_warning('yt_dlp.downloader.FragmentFD.report_retry_fragment is deprecated. '
+ 'Use yt_dlp.downloader.FileDownloader.report_retry instead')
+ return self.report_retry(err, count, retries, frag_index)
def report_skip_fragment(self, frag_index, err=None):
err = f' {err};' if err else ''
@@ -296,16 +295,23 @@ class FragmentFD(FileDownloader):
self.try_remove(ytdl_filename)
elapsed = time.time() - ctx['started']
- if ctx['tmpfilename'] == '-':
- downloaded_bytes = ctx['complete_frags_downloaded_bytes']
+ to_file = ctx['tmpfilename'] != '-'
+ if to_file:
+ downloaded_bytes = os.path.getsize(encodeFilename(ctx['tmpfilename']))
else:
+ downloaded_bytes = ctx['complete_frags_downloaded_bytes']
+
+ if not downloaded_bytes:
+ if to_file:
+ self.try_remove(ctx['tmpfilename'])
+ self.report_error('The downloaded file is empty')
+ return False
+ elif to_file:
self.try_rename(ctx['tmpfilename'], ctx['filename'])
- if self.params.get('updatetime', True):
- filetime = ctx.get('fragment_filetime')
- if filetime:
- with contextlib.suppress(Exception):
- os.utime(ctx['filename'], (time.time(), filetime))
- downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
+ filetime = ctx.get('fragment_filetime')
+ if self.params.get('updatetime', True) and filetime:
+ with contextlib.suppress(Exception):
+ os.utime(ctx['filename'], (time.time(), filetime))
self._hook_progress({
'downloaded_bytes': downloaded_bytes,
@@ -317,6 +323,7 @@ class FragmentFD(FileDownloader):
'max_progress': ctx.get('max_progress'),
'progress_idx': ctx.get('progress_idx'),
}, info_dict)
+ return True
def _prepare_external_frag_download(self, ctx):
if 'live' not in ctx:
@@ -347,6 +354,8 @@ class FragmentFD(FileDownloader):
return _key_cache[url]
def decrypt_fragment(fragment, frag_content):
+ if frag_content is None:
+ return
decrypt_info = fragment.get('decrypt_info')
if not decrypt_info or decrypt_info['METHOD'] != 'AES-128':
return frag_content
@@ -361,7 +370,7 @@ class FragmentFD(FileDownloader):
return decrypt_fragment
- def download_and_append_fragments_multiple(self, *args, pack_func=None, finish_func=None):
+ def download_and_append_fragments_multiple(self, *args, **kwargs):
'''
@params (ctx1, fragments1, info_dict1), (ctx2, fragments2, info_dict2), ...
all args must be either tuple or list
@@ -369,7 +378,7 @@ class FragmentFD(FileDownloader):
interrupt_trigger = [True]
max_progress = len(args)
if max_progress == 1:
- return self.download_and_append_fragments(*args[0], pack_func=pack_func, finish_func=finish_func)
+ return self.download_and_append_fragments(*args[0], **kwargs)
max_workers = self.params.get('concurrent_fragment_downloads', 1)
if max_progress > 1:
self._prepare_multiline_status(max_progress)
@@ -379,8 +388,7 @@ class FragmentFD(FileDownloader):
ctx['max_progress'] = max_progress
ctx['progress_idx'] = idx
return self.download_and_append_fragments(
- ctx, fragments, info_dict, pack_func=pack_func, finish_func=finish_func,
- tpe=tpe, interrupt_trigger=interrupt_trigger)
+ ctx, fragments, info_dict, **kwargs, tpe=tpe, interrupt_trigger=interrupt_trigger)
class FTPE(concurrent.futures.ThreadPoolExecutor):
# has to stop this or it's going to wait on the worker thread itself
@@ -427,18 +435,12 @@ class FragmentFD(FileDownloader):
return result
def download_and_append_fragments(
- self, ctx, fragments, info_dict, *, pack_func=None, finish_func=None,
- tpe=None, interrupt_trigger=None):
- if not interrupt_trigger:
- interrupt_trigger = (True, )
+ self, ctx, fragments, info_dict, *, is_fatal=(lambda idx: False),
+ pack_func=(lambda content, idx: content), finish_func=None,
+ tpe=None, interrupt_trigger=(True, )):
- fragment_retries = self.params.get('fragment_retries', 0)
- is_fatal = (
- ((lambda _: False) if info_dict.get('is_live') else (lambda idx: idx == 0))
- if self.params.get('skip_unavailable_fragments', True) else (lambda _: True))
-
- if not pack_func:
- pack_func = lambda frag_content, _: frag_content
+ if not self.params.get('skip_unavailable_fragments', True):
+ is_fatal = lambda _: True
def download_fragment(fragment, ctx):
if not interrupt_trigger[0]:
@@ -452,32 +454,25 @@ class FragmentFD(FileDownloader):
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
# Never skip the first fragment
- fatal, count = is_fatal(fragment.get('index') or (frag_index - 1)), 0
- while count <= fragment_retries:
+ fatal = is_fatal(fragment.get('index') or (frag_index - 1))
+
+ def error_callback(err, count, retries):
+ if fatal and count > retries:
+ ctx['dest_stream'].close()
+ self.report_retry(err, count, retries, frag_index, fatal)
+ ctx['last_error'] = err
+
+ for retry in RetryManager(self.params.get('fragment_retries'), error_callback):
try:
ctx['fragment_count'] = fragment.get('fragment_count')
- if self._download_fragment(ctx, fragment['url'], info_dict, headers):
- break
- return
+ if not self._download_fragment(ctx, fragment['url'], info_dict, headers):
+ return
except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
- # Unavailable (possibly temporary) fragments may be served.
- # First we try to retry then either skip or abort.
- # See https://github.com/ytdl-org/youtube-dl/issues/10165,
- # https://github.com/ytdl-org/youtube-dl/issues/10448).
- count += 1
- ctx['last_error'] = err
- if count <= fragment_retries:
- self.report_retry_fragment(err, frag_index, count, fragment_retries)
- except DownloadError:
- # Don't retry fragment if error occurred during HTTP downloading
- # itself since it has own retry settings
- if not fatal:
- break
- raise
-
- if count > fragment_retries and fatal:
- ctx['dest_stream'].close()
- self.report_error('Giving up after %s fragment retries' % fragment_retries)
+ retry.error = err
+ continue
+ except DownloadError: # has own retry settings
+ if fatal:
+ raise
def append_fragment(frag_content, frag_index, ctx):
if frag_content:
@@ -534,5 +529,4 @@ class FragmentFD(FileDownloader):
if finish_func is not None:
ctx['dest_stream'].write(finish_func())
ctx['dest_stream'].flush()
- self._finish_frag_download(ctx, info_dict)
- return True
+ return self._finish_frag_download(ctx, info_dict)
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index f54b3f473..4520edcd1 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -69,7 +69,7 @@ class HlsFD(FragmentFD):
elif no_crypto:
message = ('The stream has AES-128 encryption and neither ffmpeg nor pycryptodome are available; '
'Decryption will be performed natively, but will be extremely slow')
- elif re.search(r'#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
+ elif info_dict.get('extractor_key') == 'Generic' and re.search(r'(?m)#EXT-X-MEDIA-SEQUENCE:(?!0$)', s):
install_ffmpeg = '' if has_ffmpeg else 'install ffmpeg and '
message = ('Live HLS streams are not supported by the native downloader. If this is a livestream, '
f'please {install_ffmpeg}add "--downloader ffmpeg --hls-use-mpegts" to your command')
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 6b59320b8..95c870ee8 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -9,6 +9,7 @@ import urllib.error
from .common import FileDownloader
from ..utils import (
ContentTooShortError,
+ RetryManager,
ThrottledDownload,
XAttrMetadataError,
XAttrUnavailableError,
@@ -72,9 +73,6 @@ class HttpFD(FileDownloader):
ctx.is_resume = ctx.resume_len > 0
- count = 0
- retries = self.params.get('retries', 0)
-
class SucceedDownload(Exception):
pass
@@ -206,6 +204,12 @@ class HttpFD(FileDownloader):
except RESPONSE_READ_EXCEPTIONS as err:
raise RetryDownload(err)
+ def close_stream():
+ if ctx.stream is not None:
+ if not ctx.tmpfilename == '-':
+ ctx.stream.close()
+ ctx.stream = None
+
def download():
data_len = ctx.data.info().get('Content-length', None)
@@ -239,12 +243,9 @@ class HttpFD(FileDownloader):
before = start # start measuring
def retry(e):
- to_stdout = ctx.tmpfilename == '-'
- if ctx.stream is not None:
- if not to_stdout:
- ctx.stream.close()
- ctx.stream = None
- ctx.resume_len = byte_counter if to_stdout else os.path.getsize(encodeFilename(ctx.tmpfilename))
+ close_stream()
+ ctx.resume_len = (byte_counter if ctx.tmpfilename == '-'
+ else os.path.getsize(encodeFilename(ctx.tmpfilename)))
raise RetryDownload(e)
while True:
@@ -346,9 +347,7 @@ class HttpFD(FileDownloader):
if data_len is not None and byte_counter != data_len:
err = ContentTooShortError(byte_counter, int(data_len))
- if count <= retries:
- retry(err)
- raise err
+ retry(err)
self.try_rename(ctx.tmpfilename, ctx.filename)
@@ -367,21 +366,20 @@ class HttpFD(FileDownloader):
return True
- while count <= retries:
+ for retry in RetryManager(self.params.get('retries'), self.report_retry):
try:
establish_connection()
return download()
- except RetryDownload as e:
- count += 1
- if count <= retries:
- self.report_retry(e.source_error, count, retries)
- else:
- self.to_screen(f'[download] Got server HTTP error: {e.source_error}')
+ except RetryDownload as err:
+ retry.error = err.source_error
continue
except NextFragment:
+ retry.error = None
+ retry.attempt -= 1
continue
except SucceedDownload:
return True
-
- self.report_error('giving up after %s retries' % retries)
+ except: # noqa: E722
+ close_stream()
+ raise
return False
diff --git a/yt_dlp/downloader/ism.py b/yt_dlp/downloader/ism.py
index 8a0071ab3..a157a8ad9 100644
--- a/yt_dlp/downloader/ism.py
+++ b/yt_dlp/downloader/ism.py
@@ -5,6 +5,7 @@ import time
import urllib.error
from .fragment import FragmentFD
+from ..utils import RetryManager
u8 = struct.Struct('>B')
u88 = struct.Struct('>Bx')
@@ -137,6 +138,8 @@ def write_piff_header(stream, params):
if fourcc == 'AACL':
sample_entry_box = box(b'mp4a', sample_entry_payload)
+ if fourcc == 'EC-3':
+ sample_entry_box = box(b'ec-3', sample_entry_payload)
elif stream_type == 'video':
sample_entry_payload += u16.pack(0) # pre defined
sample_entry_payload += u16.pack(0) # reserved
@@ -245,7 +248,6 @@ class IsmFD(FragmentFD):
'ism_track_written': False,
})
- fragment_retries = self.params.get('fragment_retries', 0)
skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
frag_index = 0
@@ -253,8 +255,10 @@ class IsmFD(FragmentFD):
frag_index += 1
if frag_index <= ctx['fragment_index']:
continue
- count = 0
- while count <= fragment_retries:
+
+ retry_manager = RetryManager(self.params.get('fragment_retries'), self.report_retry,
+ frag_index=frag_index, fatal=not skip_unavailable_fragments)
+ for retry in retry_manager:
try:
success = self._download_fragment(ctx, segment['url'], info_dict)
if not success:
@@ -267,18 +271,13 @@ class IsmFD(FragmentFD):
write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
extra_state['ism_track_written'] = True
self._append_fragment(ctx, frag_content)
- break
except urllib.error.HTTPError as err:
- count += 1
- if count <= fragment_retries:
- self.report_retry_fragment(err, frag_index, count, fragment_retries)
- if count > fragment_retries:
- if skip_unavailable_fragments:
- self.report_skip_fragment(frag_index)
+ retry.error = err
continue
- self.report_error('giving up after %s fragment retries' % fragment_retries)
- return False
- self._finish_frag_download(ctx, info_dict)
+ if retry_manager.error:
+ if not skip_unavailable_fragments:
+ return False
+ self.report_skip_fragment(frag_index)
- return True
+ return self._finish_frag_download(ctx, info_dict)
diff --git a/yt_dlp/downloader/mhtml.py b/yt_dlp/downloader/mhtml.py
index ce2d39947..d977dcec3 100644
--- a/yt_dlp/downloader/mhtml.py
+++ b/yt_dlp/downloader/mhtml.py
@@ -4,6 +4,7 @@ import re
import uuid
from .fragment import FragmentFD
+from ..compat import imghdr
from ..utils import escapeHTML, formatSeconds, srt_subtitles_timecode, urljoin
from ..version import __version__ as YT_DLP_VERSION
@@ -166,21 +167,13 @@ body > figure > img {
continue
frag_content = self._read_fragment(ctx)
- mime_type = b'image/jpeg'
- if frag_content.startswith(b'\x89PNG\r\n\x1a\n'):
- mime_type = b'image/png'
- if frag_content.startswith((b'GIF87a', b'GIF89a')):
- mime_type = b'image/gif'
- if frag_content.startswith(b'RIFF') and frag_content[8:12] == b'WEBP':
- mime_type = b'image/webp'
-
frag_header = io.BytesIO()
frag_header.write(
b'--%b\r\n' % frag_boundary.encode('us-ascii'))
frag_header.write(
b'Content-ID: <%b>\r\n' % self._gen_cid(i, fragment, frag_boundary).encode('us-ascii'))
frag_header.write(
- b'Content-type: %b\r\n' % mime_type)
+ b'Content-type: %b\r\n' % f'image/{imghdr.what(h=frag_content) or "jpeg"}'.encode())
frag_header.write(
b'Content-length: %u\r\n' % len(frag_content))
frag_header.write(
@@ -193,5 +186,4 @@ body > figure > img {
ctx['dest_stream'].write(
b'--%b--\r\n\r\n' % frag_boundary.encode('us-ascii'))
- self._finish_frag_download(ctx, info_dict)
- return True
+ return self._finish_frag_download(ctx, info_dict)
diff --git a/yt_dlp/downloader/websocket.py b/yt_dlp/downloader/websocket.py
index 727a15828..6837ff1da 100644
--- a/yt_dlp/downloader/websocket.py
+++ b/yt_dlp/downloader/websocket.py
@@ -1,3 +1,4 @@
+import asyncio
import contextlib
import os
import signal
@@ -5,7 +6,6 @@ import threading
from .common import FileDownloader
from .external import FFmpegFD
-from ..compat import asyncio
from ..dependencies import websockets
diff --git a/yt_dlp/downloader/youtube_live_chat.py b/yt_dlp/downloader/youtube_live_chat.py
index 5334c6c95..5928fecf0 100644
--- a/yt_dlp/downloader/youtube_live_chat.py
+++ b/yt_dlp/downloader/youtube_live_chat.py
@@ -3,7 +3,13 @@ import time
import urllib.error
from .fragment import FragmentFD
-from ..utils import RegexNotFoundError, dict_get, int_or_none, try_get
+from ..utils import (
+ RegexNotFoundError,
+ RetryManager,
+ dict_get,
+ int_or_none,
+ try_get,
+)
class YoutubeLiveChatFD(FragmentFD):
@@ -16,7 +22,6 @@ class YoutubeLiveChatFD(FragmentFD):
self.report_warning('Live chat download runs until the livestream ends. '
'If you wish to download the video simultaneously, run a separate yt-dlp instance')
- fragment_retries = self.params.get('fragment_retries', 0)
test = self.params.get('test', False)
ctx = {
@@ -104,8 +109,7 @@ class YoutubeLiveChatFD(FragmentFD):
return continuation_id, live_offset, click_tracking_params
def download_and_parse_fragment(url, frag_index, request_data=None, headers=None):
- count = 0
- while count <= fragment_retries:
+ for retry in RetryManager(self.params.get('fragment_retries'), self.report_retry, frag_index=frag_index):
try:
success = dl_fragment(url, request_data, headers)
if not success:
@@ -120,21 +124,15 @@ class YoutubeLiveChatFD(FragmentFD):
live_chat_continuation = try_get(
data,
lambda x: x['continuationContents']['liveChatContinuation'], dict) or {}
- if info_dict['protocol'] == 'youtube_live_chat_replay':
- if frag_index == 1:
- continuation_id, offset, click_tracking_params = try_refresh_replay_beginning(live_chat_continuation)
- else:
- continuation_id, offset, click_tracking_params = parse_actions_replay(live_chat_continuation)
- elif info_dict['protocol'] == 'youtube_live_chat':
- continuation_id, offset, click_tracking_params = parse_actions_live(live_chat_continuation)
- return True, continuation_id, offset, click_tracking_params
+
+ func = (info_dict['protocol'] == 'youtube_live_chat' and parse_actions_live
+ or frag_index == 1 and try_refresh_replay_beginning
+ or parse_actions_replay)
+ return (True, *func(live_chat_continuation))
except urllib.error.HTTPError as err:
- count += 1
- if count <= fragment_retries:
- self.report_retry_fragment(err, frag_index, count, fragment_retries)
- if count > fragment_retries:
- self.report_error('giving up after %s fragment retries' % fragment_retries)
- return False, None, None, None
+ retry.error = err
+ continue
+ return False, None, None, None
self._prepare_and_start_frag_download(ctx, info_dict)
@@ -193,8 +191,7 @@ class YoutubeLiveChatFD(FragmentFD):
if test:
break
- self._finish_frag_download(ctx, info_dict)
- return True
+ return self._finish_frag_download(ctx, info_dict)
@staticmethod
def parse_live_timestamp(action):
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-02 19:59:04 +0000