From 86e5f3ed2e6e71eb81ea4c9e26288f16119ffd0c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Apr 2022 20:40:28 +0530 Subject: [cleanup] Upgrade syntax Using https://github.com/asottile/pyupgrade 1. `__future__` imports and `coding: utf-8` were removed 2. Files were rewritten with `pyupgrade --py36-plus --keep-percent-format` 3. f-strings were cherry-picked from `pyupgrade --py36-plus` Extractors are left untouched (except removing header) to avoid unnecessary merge conflicts --- yt_dlp/downloader/external.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 71af705ea..b6dd32701 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import os.path import re import subprocess @@ -56,7 +54,7 @@ class ExternalFD(FragmentFD): } if filename != '-': fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize)) + self.to_screen(f'\r[{self.get_basename()}] Downloaded {fsize} bytes') self.try_rename(tmpfilename, filename) status.update({ 'downloaded_bytes': fsize, @@ -157,7 +155,7 @@ class ExternalFD(FragmentFD): fragment_filename = '%s-Frag%d' % (tmpfilename, frag_index) try: src, _ = self.sanitize_open(fragment_filename, 'rb') - except IOError as err: + except OSError as err: if skip_unavailable_fragments and frag_index > 1: self.report_skip_fragment(frag_index, err) continue @@ -179,7 +177,7 @@ class CurlFD(ExternalFD): cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed'] if info_dict.get('http_headers') is not None: for key, val in info_dict['http_headers'].items(): - cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--header', f'{key}: {val}'] cmd += self._bool_option('--continue-at', 'continuedl', '-', '0') cmd += self._valueless_option('--silent', 'noprogress') @@ -216,7 +214,7 @@ class AxelFD(ExternalFD): cmd = [self.exe, '-o', tmpfilename] if info_dict.get('http_headers') is not None: for key, val in info_dict['http_headers'].items(): - cmd += ['-H', '%s: %s' % (key, val)] + cmd += ['-H', f'{key}: {val}'] cmd += self._configuration_args() cmd += ['--', info_dict['url']] return cmd @@ -229,7 +227,7 @@ class WgetFD(ExternalFD): cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies', '--compression=auto'] if info_dict.get('http_headers') is not None: for key, val in info_dict['http_headers'].items(): - cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--header', f'{key}: {val}'] cmd += self._option('--limit-rate', 'ratelimit') retry = self._option('--tries', 'retries') if len(retry) == 2: @@ -240,7 +238,7 @@ class WgetFD(ExternalFD): proxy = self.params.get('proxy') if proxy: for var in ('http_proxy', 'https_proxy'): - cmd += ['--execute', '%s=%s' % (var, proxy)] + cmd += ['--execute', f'{var}={proxy}'] cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate') cmd += self._configuration_args() cmd += ['--', info_dict['url']] @@ -271,7 +269,7 @@ class Aria2cFD(ExternalFD): if info_dict.get('http_headers') is not None: for key, val in info_dict['http_headers'].items(): - cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--header', f'{key}: {val}'] cmd += self._option('--max-overall-download-limit', 'ratelimit') cmd += self._option('--interface', 'source_address') cmd += self._option('--all-proxy', 'proxy') @@ -289,10 +287,10 @@ class Aria2cFD(ExternalFD): dn = os.path.dirname(tmpfilename) if dn: if not os.path.isabs(dn): - dn = '.%s%s' % (os.path.sep, dn) + dn = f'.{os.path.sep}{dn}' cmd += ['--dir', dn + os.path.sep] if 'fragments' not in info_dict: - cmd += ['--out', '.%s%s' % (os.path.sep, os.path.basename(tmpfilename))] + cmd += ['--out', f'.{os.path.sep}{os.path.basename(tmpfilename)}'] cmd += ['--auto-file-renaming=false'] if 'fragments' in info_dict: @@ -320,7 +318,7 @@ class HttpieFD(ExternalFD): if info_dict.get('http_headers') is not None: for key, val in info_dict['http_headers'].items(): - cmd += ['%s:%s' % (key, val)] + cmd += [f'{key}:{val}'] return cmd @@ -393,7 +391,7 @@ class FFmpegFD(ExternalFD): headers = handle_youtubedl_headers(info_dict['http_headers']) args += [ '-headers', - ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())] + ''.join(f'{key}: {val}\r\n' for key, val in headers.items())] env = None proxy = self.params.get('proxy') -- cgit v1.2.3 From f82711587cee043cb2496fe180b5cc0e07c06eda Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 04:02:57 +0530 Subject: [cleanup] Sort imports Using https://github.com/PyCQA/isort isort -m VERTICAL_HANGING_INDENT --py 36 -l 80 --rr -n --tc . --- yt_dlp/downloader/external.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index b6dd32701..6c5616c60 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -5,23 +5,20 @@ import sys import time from .fragment import FragmentFD -from ..compat import ( - compat_setenv, - compat_str, -) -from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS +from ..compat import compat_setenv, compat_str +from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( + Popen, + _configuration_args, + check_executable, classproperty, + cli_bool_option, cli_option, cli_valueless_option, - cli_bool_option, - _configuration_args, determine_ext, - encodeFilename, encodeArgument, + encodeFilename, handle_youtubedl_headers, - check_executable, - Popen, remove_end, ) -- cgit v1.2.3 From 00828e2c9311b90d317fa054883dd63e21fffa78 Mon Sep 17 00:00:00 2001 From: Elyse Date: Tue, 26 Apr 2022 04:54:56 -0500 Subject: [downloader/ffmpeg] Specify headers for each URL (#3553) Closes #2696 Authored by: elyse0 --- yt_dlp/downloader/external.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 6c5616c60..da38e502d 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -382,13 +382,15 @@ class FFmpegFD(ExternalFD): # if end_time: # args += ['-t', compat_str(end_time - start_time)] - if info_dict.get('http_headers') is not None and re.match(r'^https?://', urls[0]): - # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv: - # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header. - headers = handle_youtubedl_headers(info_dict['http_headers']) - args += [ + http_headers = None + if info_dict.get('http_headers'): + youtubedl_headers = handle_youtubedl_headers(info_dict['http_headers']) + http_headers = [ + # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv: + # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header. '-headers', - ''.join(f'{key}: {val}\r\n' for key, val in headers.items())] + ''.join(f'{key}: {val}\r\n' for key, val in youtubedl_headers.items()) + ] env = None proxy = self.params.get('proxy') @@ -441,6 +443,11 @@ class FFmpegFD(ExternalFD): args += ['-rtmp_conn', conn] for i, url in enumerate(urls): + # We need to specify headers for each http input stream + # otherwise, it will only be applied to the first. + # https://github.com/yt-dlp/yt-dlp/issues/2696 + if http_headers is not None and re.match(r'^https?://', url): + args += http_headers args += self._configuration_args((f'_i{i + 1}', '_i')) + ['-i', url] args += ['-c', 'copy'] -- cgit v1.2.3 From 0a5a191a2a33e3b305aaf684576b7129ba5173a0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 27 Apr 2022 21:52:57 +0530 Subject: Improve `--clean-infojson` It should not removes fields that may be needed for `--load-infojson`. Eg: `_ffmpeg_args`, `_has_drm` --- yt_dlp/downloader/external.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index da38e502d..4fe56bb95 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -20,6 +20,7 @@ from ..utils import ( encodeFilename, handle_youtubedl_headers, remove_end, + traverse_obj, ) @@ -363,9 +364,11 @@ class FFmpegFD(ExternalFD): if not self.params.get('verbose'): args += ['-hide_banner'] - args += info_dict.get('_ffmpeg_args', []) + args += traverse_obj(info_dict, ('downloader_options', 'ffmpeg_args'), default=[]) - # This option exists only for compatibility. Extractors should use `_ffmpeg_args` instead + # These exists only for compatibility. Extractors should use + # info_dict['downloader_options']['ffmpeg_args'] instead + args += info_dict.get('_ffmpeg_args') seekable = info_dict.get('_seekable') if seekable is not None: # setting -seekable prevents ffmpeg from guessing if the server -- cgit v1.2.3 From 1d485a1a799bbeeb2faea0595676ca7d4c0f3716 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 29 Apr 2022 07:18:36 +0530 Subject: [cleanup] Misc fixes Closes #3565, https://github.com/yt-dlp/yt-dlp/issues/3514#issuecomment-1105944364 --- yt_dlp/downloader/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 4fe56bb95..4f9f8f6e5 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -368,7 +368,7 @@ class FFmpegFD(ExternalFD): # These exists only for compatibility. Extractors should use # info_dict['downloader_options']['ffmpeg_args'] instead - args += info_dict.get('_ffmpeg_args') + args += info_dict.get('_ffmpeg_args') or [] seekable = info_dict.get('_seekable') if seekable is not None: # setting -seekable prevents ffmpeg from guessing if the server -- cgit v1.2.3 From 0f06bcd7591332937fdec497d6cbb4914358bc79 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 9 May 2022 17:24:28 +0530 Subject: [cleanup] Minor fixes (See desc) * [youtube] Fix `--youtube-skip-dash-manifest` * [build] Use `$()` in `Makefile`. Closes #3684 * Fix bug in 385ffb467b2285e85a2a5495b90314ba1f8e0700 * Fix bug in 43d7f5a5d0c77556156a3f8caa6976d3908a1e38 * [cleanup] Remove unnecessary `utf-8` from `str.encode`/`bytes.decode` * [utils] LazyList: Expose unnecessarily "protected" attributes and other minor cleanup --- yt_dlp/downloader/external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'yt_dlp/downloader/external.py') diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index 4f9f8f6e5..85c6a6977 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -299,7 +299,7 @@ class Aria2cFD(ExternalFD): fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index) url_list.append('%s\n\tout=%s' % (fragment['url'], fragment_filename)) stream, _ = self.sanitize_open(url_list_file, 'wb') - stream.write('\n'.join(url_list).encode('utf-8')) + stream.write('\n'.join(url_list).encode()) stream.close() cmd += ['-i', url_list_file] else: -- cgit v1.2.3