diff options
| -rw-r--r-- | yt_dlp/YoutubeDL.py | 12 | ||||
| -rw-r--r-- | yt_dlp/__init__.py | 6 | ||||
| -rw-r--r-- | yt_dlp/downloader/common.py | 6 | ||||
| -rw-r--r-- | yt_dlp/downloader/http.py | 14 | ||||
| -rw-r--r-- | yt_dlp/options.py | 4 | ||||
| -rw-r--r-- | yt_dlp/utils.py | 5 |
6 files changed, 41 insertions, 6 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index aa93b6d1d..ffc72ba5d 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -101,6 +101,7 @@ from .utils import ( str_or_none, strftime_or_none, subtitles_filename, + ThrottledDownload, to_high_limit_path, traverse_obj, UnavailableVideoError, @@ -398,10 +399,9 @@ class YoutubeDL(object): The following parameters are not used by YoutubeDL itself, they are used by the downloader (see yt_dlp/downloader/common.py): - nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test, - noresizebuffer, retries, continuedl, noprogress, consoletitle, - xattr_set_filesize, external_downloader_args, hls_use_mpegts, - http_chunk_size. + nopart, updatetime, buffersize, ratelimit, throttledratelimit, min_filesize, + max_filesize, test, noresizebuffer, retries, continuedl, noprogress, consoletitle, + xattr_set_filesize, external_downloader_args, hls_use_mpegts, http_chunk_size. The following options are used by the post processors: prefer_ffmpeg: If False, use avconv instead of ffmpeg if both are available, @@ -1145,6 +1145,10 @@ class YoutubeDL(object): self.report_error(msg) except ExtractorError as e: # An error we somewhat expected self.report_error(compat_str(e), e.format_traceback()) + except ThrottledDownload: + self.to_stderr('\r') + self.report_warning('The download speed is below throttle limit. Re-extracting data') + return wrapper(self, *args, **kwargs) except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached): raise except Exception as e: diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 728b3321f..21b45db0a 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -151,6 +151,11 @@ def _real_main(argv=None): if numeric_limit is None: parser.error('invalid rate limit specified') opts.ratelimit = numeric_limit + if opts.throttledratelimit is not None: + numeric_limit = FileDownloader.parse_bytes(opts.throttledratelimit) + if numeric_limit is None: + parser.error('invalid rate limit specified') + opts.throttledratelimit = numeric_limit if opts.min_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.min_filesize) if numeric_limit is None: @@ -552,6 +557,7 @@ def _real_main(argv=None): 'ignoreerrors': opts.ignoreerrors, 'force_generic_extractor': opts.force_generic_extractor, 'ratelimit': opts.ratelimit, + 'throttledratelimit': opts.throttledratelimit, 'overwrites': opts.overwrites, 'retries': opts.retries, 'fragment_retries': opts.fragment_retries, diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index 66e9677ed..65751bb3b 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -14,6 +14,7 @@ from ..utils import ( format_bytes, shell_quote, timeconvert, + ThrottledDownload, ) @@ -32,6 +33,7 @@ class FileDownloader(object): verbose: Print additional info to stdout. quiet: Do not print messages to stdout. ratelimit: Download speed limit, in bytes/sec. + throttledratelimit: Assume the download is being throttled below this speed (bytes/sec) retries: Number of times to retry for HTTP error 5xx buffersize: Size of download buffer in bytes. noresizebuffer: Do not automatically resize the download buffer. @@ -170,7 +172,7 @@ class FileDownloader(object): def slow_down(self, start_time, now, byte_counter): """Sleep if the download speed is over the rate limit.""" rate_limit = self.params.get('ratelimit') - if rate_limit is None or byte_counter == 0: + if byte_counter == 0: return if now is None: now = time.time() @@ -178,7 +180,7 @@ class FileDownloader(object): if elapsed <= 0.0: return speed = float(byte_counter) / elapsed - if speed > rate_limit: + if rate_limit is not None and speed > rate_limit: sleep_time = float(byte_counter) / rate_limit - elapsed if sleep_time > 0: time.sleep(sleep_time) diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py index bf77f4427..15eb54aab 100644 --- a/yt_dlp/downloader/http.py +++ b/yt_dlp/downloader/http.py @@ -18,6 +18,7 @@ from ..utils import ( int_or_none, sanitize_open, sanitized_Request, + ThrottledDownload, write_xattr, XAttrMetadataError, XAttrUnavailableError, @@ -223,6 +224,7 @@ class HttpFD(FileDownloader): # measure time over whole while-loop, so slow_down() and best_block_size() work together properly now = None # needed for slow_down() in the first loop run before = start # start measuring + throttle_start = None def retry(e): to_stdout = ctx.tmpfilename == '-' @@ -313,6 +315,18 @@ class HttpFD(FileDownloader): if data_len is not None and byte_counter == data_len: break + if speed and speed < (self.params.get('throttledratelimit') or 0): + # The speed must stay below the limit for 3 seconds + # This prevents raising error when the speed temporarily goes down + if throttle_start is None: + throttle_start = now + elif now - throttle_start > 3: + if ctx.stream is not None and ctx.tmpfilename != '-': + ctx.stream.close() + raise ThrottledDownload() + else: + throttle_start = None + if not is_test and ctx.chunk_size and ctx.data_len is not None and byte_counter < ctx.data_len: ctx.resume_len = byte_counter # ctx.block_size = block_size diff --git a/yt_dlp/options.py b/yt_dlp/options.py index 535178627..bd817fed7 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -600,6 +600,10 @@ def parseOpts(overrideArguments=None): dest='ratelimit', metavar='RATE', help='Maximum download rate in bytes per second (e.g. 50K or 4.2M)') downloader.add_option( + '--throttled-rate', + dest='throttledratelimit', metavar='RATE', + help='Minimum download rate in bytes per second below which throttling is assumed and the video data is re-extracted (e.g. 100K)') + downloader.add_option( '-R', '--retries', dest='retries', metavar='RETRIES', default=10, help='Number of retries (default is %default), or "infinite"') diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 8e85620cc..c9599af53 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -2504,6 +2504,11 @@ class RejectedVideoReached(YoutubeDLError): pass +class ThrottledDownload(YoutubeDLError): + """ Download speed below --throttled-rate. """ + pass + + class MaxDownloadsReached(YoutubeDLError): """ --max-downloads limit has been reached. """ pass |