diff options
Diffstat (limited to 'yt_dlp')
44 files changed, 321 insertions, 458 deletions
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 56f0346dc..a8bb7f45c 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -12,6 +12,7 @@ import locale import operator import os import platform +import random import re import shutil import subprocess @@ -20,13 +21,12 @@ import tempfile import time import tokenize import traceback -import random import unicodedata import urllib.request - from enum import Enum from string import ascii_letters +from .cache import Cache from .compat import ( compat_brotli, compat_get_terminal_size, @@ -39,74 +39,100 @@ from .compat import ( windows_enable_vt_mode, ) from .cookies import load_cookies +from .downloader import FFmpegFD, get_suitable_downloader, shorten_protocol_name +from .downloader.rtmp import rtmpdump_version +from .extractor import _LAZY_LOADER +from .extractor import _PLUGIN_CLASSES as plugin_extractors +from .extractor import gen_extractor_classes, get_info_extractor +from .extractor.openload import PhantomJSwrapper +from .minicurses import format_text +from .postprocessor import _PLUGIN_CLASSES as plugin_postprocessors +from .postprocessor import ( + EmbedThumbnailPP, + FFmpegFixupDuplicateMoovPP, + FFmpegFixupDurationPP, + FFmpegFixupM3u8PP, + FFmpegFixupM4aPP, + FFmpegFixupStretchedPP, + FFmpegFixupTimestampPP, + FFmpegMergerPP, + FFmpegPostProcessor, + MoveFilesAfterDownloadPP, + get_postprocessor, +) +from .update import detect_variant from .utils import ( + DEFAULT_OUTTMPL, + LINK_TEMPLATES, + NO_DEFAULT, + OUTTMPL_TYPES, + POSTPROCESS_WHEN, + STR_FORMAT_RE_TMPL, + STR_FORMAT_TYPES, + ContentTooShortError, + DateRange, + DownloadCancelled, + DownloadError, + EntryNotInPlaylist, + ExistingVideoReached, + ExtractorError, + GeoRestrictedError, + HEADRequest, + InAdvancePagedList, + ISO3166Utils, + LazyList, + MaxDownloadsReached, + PagedList, + PerRequestProxyHandler, + Popen, + PostProcessingError, + ReExtractInfo, + RejectedVideoReached, + SameFileError, + UnavailableVideoError, + YoutubeDLCookieProcessor, + YoutubeDLHandler, + YoutubeDLRedirectHandler, age_restricted, args_to_str, - ContentTooShortError, date_from_str, - DateRange, - DEFAULT_OUTTMPL, determine_ext, determine_protocol, - DownloadCancelled, - DownloadError, encode_compat_str, encodeFilename, - EntryNotInPlaylist, error_to_compat_str, - ExistingVideoReached, expand_path, - ExtractorError, filter_dict, float_or_none, format_bytes, - format_field, format_decimal_suffix, + format_field, formatSeconds, - GeoRestrictedError, get_domain, has_certifi, - HEADRequest, - InAdvancePagedList, int_or_none, iri_to_uri, - ISO3166Utils, join_nonempty, - LazyList, - LINK_TEMPLATES, locked_file, make_dir, make_HTTPS_handler, - MaxDownloadsReached, merge_headers, network_exceptions, - NO_DEFAULT, number_of_digits, orderedSet, - OUTTMPL_TYPES, - PagedList, parse_filesize, - PerRequestProxyHandler, platform_name, - Popen, - POSTPROCESS_WHEN, - PostProcessingError, preferredencoding, prepend_extension, - ReExtractInfo, register_socks_protocols, - RejectedVideoReached, remove_terminal_sequences, render_table, replace_extension, - SameFileError, sanitize_filename, sanitize_path, sanitize_url, sanitized_Request, std_headers, - STR_FORMAT_RE_TMPL, - STR_FORMAT_TYPES, str_or_none, strftime_or_none, subtitles_filename, @@ -115,47 +141,13 @@ from .utils import ( to_high_limit_path, traverse_obj, try_get, - UnavailableVideoError, url_basename, variadic, version_tuple, write_json_file, write_string, - YoutubeDLCookieProcessor, - YoutubeDLHandler, - YoutubeDLRedirectHandler, -) -from .cache import Cache -from .minicurses import format_text -from .extractor import ( - gen_extractor_classes, - get_info_extractor, - _LAZY_LOADER, - _PLUGIN_CLASSES as plugin_extractors -) -from .extractor.openload import PhantomJSwrapper -from .downloader import ( - FFmpegFD, - get_suitable_downloader, - shorten_protocol_name -) -from .downloader.rtmp import rtmpdump_version -from .postprocessor import ( - get_postprocessor, - EmbedThumbnailPP, - FFmpegFixupDuplicateMoovPP, - FFmpegFixupDurationPP, - FFmpegFixupM3u8PP, - FFmpegFixupM4aPP, - FFmpegFixupStretchedPP, - FFmpegFixupTimestampPP, - FFmpegMergerPP, - FFmpegPostProcessor, - MoveFilesAfterDownloadPP, - _PLUGIN_CLASSES as plugin_postprocessors ) -from .update import detect_variant -from .version import __version__, RELEASE_GIT_HEAD +from .version import RELEASE_GIT_HEAD, __version__ if compat_os_name == 'nt': import ctypes @@ -3666,9 +3658,9 @@ class YoutubeDL: ) or 'none' write_debug('exe versions: %s' % exe_str) + from .cookies import SECRETSTORAGE_AVAILABLE, SQLITE_AVAILABLE from .downloader.websocket import has_websockets from .postprocessor.embedthumbnail import has_mutagen - from .cookies import SQLITE_AVAILABLE, SECRETSTORAGE_AVAILABLE lib_str = join_nonempty( compat_brotli and compat_brotli.__name__, diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py index 91bf5c4ce..f339e4cd1 100644 --- a/yt_dlp/__init__.py +++ b/yt_dlp/__init__.py @@ -9,48 +9,44 @@ import random import re import sys +from .compat import compat_getpass, compat_os_name, compat_shlex_quote +from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS +from .downloader import FileDownloader +from .extractor import gen_extractors, list_extractors +from .extractor.adobepass import MSO_INFO +from .extractor.common import InfoExtractor from .options import parseOpts -from .compat import ( - compat_getpass, - compat_os_name, - compat_shlex_quote, +from .postprocessor import ( + FFmpegExtractAudioPP, + FFmpegSubtitlesConvertorPP, + FFmpegThumbnailsConvertorPP, + FFmpegVideoConvertorPP, + FFmpegVideoRemuxerPP, + MetadataFromFieldPP, + MetadataParserPP, ) -from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS +from .update import run_update from .utils import ( + NO_DEFAULT, DateRange, - decodeOption, DownloadCancelled, DownloadError, + GeoUtils, + SameFileError, + decodeOption, expand_path, float_or_none, - GeoUtils, int_or_none, match_filter_func, - NO_DEFAULT, parse_duration, preferredencoding, read_batch_urls, render_table, - SameFileError, setproctitle, std_headers, traverse_obj, write_string, ) -from .update import run_update -from .downloader import FileDownloader -from .extractor import gen_extractors, list_extractors -from .extractor.common import InfoExtractor -from .extractor.adobepass import MSO_INFO -from .postprocessor import ( - FFmpegExtractAudioPP, - FFmpegSubtitlesConvertorPP, - FFmpegThumbnailsConvertorPP, - FFmpegVideoConvertorPP, - FFmpegVideoRemuxerPP, - MetadataFromFieldPP, - MetadataParserPP, -) from .YoutubeDL import YoutubeDL diff --git a/yt_dlp/aes.py b/yt_dlp/aes.py index e5d73f740..01818df61 100644 --- a/yt_dlp/aes.py +++ b/yt_dlp/aes.py @@ -1,15 +1,7 @@ from math import ceil -from .compat import ( - compat_b64decode, - compat_ord, - compat_pycrypto_AES, -) -from .utils import ( - bytes_to_intlist, - intlist_to_bytes, -) - +from .compat import compat_b64decode, compat_ord, compat_pycrypto_AES +from .utils import bytes_to_intlist, intlist_to_bytes if compat_pycrypto_AES: def aes_cbc_decrypt_bytes(data, key, iv): diff --git a/yt_dlp/cache.py b/yt_dlp/cache.py index f93ef85e7..0cac3ee88 100644 --- a/yt_dlp/cache.py +++ b/yt_dlp/cache.py @@ -6,10 +6,7 @@ import shutil import traceback from .compat import compat_getenv -from .utils import ( - expand_path, - write_json_file, -) +from .utils import expand_path, write_json_file class Cache: diff --git a/yt_dlp/cookies.py b/yt_dlp/cookies.py index 1d92fd8ce..6ff9f6f2d 100644 --- a/yt_dlp/cookies.py +++ b/yt_dlp/cookies.py @@ -16,17 +16,9 @@ from .aes import ( aes_gcm_decrypt_and_verify_bytes, unpad_pkcs7, ) -from .compat import ( - compat_b64decode, - compat_cookiejar_Cookie, -) +from .compat import compat_b64decode, compat_cookiejar_Cookie from .minicurses import MultilinePrinter, QuietMultilinePrinter -from .utils import ( - error_to_str, - expand_path, - Popen, - YoutubeDLCookieJar, -) +from .utils import Popen, YoutubeDLCookieJar, error_to_str, expand_path try: import sqlite3 diff --git a/yt_dlp/downloader/__init__.py b/yt_dlp/downloader/__init__.py index f5abfd5df..5aba303dd 100644 --- a/yt_dlp/downloader/__init__.py +++ b/yt_dlp/downloader/__init__.py @@ -1,8 +1,5 @@ from ..compat import compat_str -from ..utils import ( - determine_protocol, - NO_DEFAULT -) +from ..utils import NO_DEFAULT, determine_protocol def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None, to_stdout=False): @@ -27,21 +24,18 @@ def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=N # Some of these require get_suitable_downloader from .common import FileDownloader from .dash import DashSegmentsFD +from .external import FFmpegFD, get_external_downloader from .f4m import F4mFD from .fc2 import FC2LiveFD from .hls import HlsFD from .http import HttpFD -from .rtmp import RtmpFD -from .rtsp import RtspFD from .ism import IsmFD from .mhtml import MhtmlFD from .niconico import NiconicoDmcFD +from .rtmp import RtmpFD +from .rtsp import RtspFD from .websocket import WebSocketFragmentFD from .youtube_live_chat import YoutubeLiveChatFD -from .external import ( - get_external_downloader, - FFmpegFD, -) PROTOCOL_MAP = { 'rtmp': RtmpFD, diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py index d42539931..3033926ae 100644 --- a/yt_dlp/downloader/common.py +++ b/yt_dlp/downloader/common.py @@ -1,26 +1,26 @@ +import errno import os +import random import re import time -import random -import errno +from ..minicurses import ( + BreaklineStatusPrinter, + MultilineLogger, + MultilinePrinter, + QuietMultilinePrinter, +) from ..utils import ( + LockingUnsupportedError, decodeArgument, encodeFilename, error_to_compat_str, format_bytes, - LockingUnsupportedError, sanitize_open, shell_quote, timeconvert, timetuple_from_msec, ) -from ..minicurses import ( - MultilineLogger, - MultilinePrinter, - QuietMultilinePrinter, - BreaklineStatusPrinter -) class FileDownloader: diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py index 64eb5e66a..e6efae485 100644 --- a/yt_dlp/downloader/dash.py +++ b/yt_dlp/downloader/dash.py @@ -1,8 +1,7 @@ import time -from ..downloader import get_suitable_downloader from .fragment import FragmentFD - +from ..downloader import get_suitable_downloader from ..utils import urljoin diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py index b6dd32701..6c5616c60 100644 --- a/yt_dlp/downloader/external.py +++ b/yt_dlp/downloader/external.py @@ -5,23 +5,20 @@ import sys import time from .fragment import FragmentFD -from ..compat import ( - compat_setenv, - compat_str, -) -from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS +from ..compat import compat_setenv, compat_str +from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor from ..utils import ( + Popen, + _configuration_args, + check_executable, classproperty, + cli_bool_option, cli_option, cli_valueless_option, - cli_bool_option, - _configuration_args, determine_ext, - encodeFilename, encodeArgument, + encodeFilename, handle_youtubedl_headers, - check_executable, - Popen, remove_end, ) diff --git a/yt_dlp/downloader/f4m.py b/yt_dlp/downloader/f4m.py index 414071075..12ecec008 100644 --- a/yt_dlp/downloader/f4m.py +++ b/yt_dlp/downloader/f4m.py @@ -6,16 +6,13 @@ from .fragment import FragmentFD from ..compat import ( compat_b64decode, compat_etree_fromstring, - compat_urlparse, - compat_urllib_error, - compat_urllib_parse_urlparse, compat_struct_pack, compat_struct_unpack, + compat_urllib_error, + compat_urllib_parse_urlparse, + compat_urlparse, ) -from ..utils import ( - fix_xml_ampersands, - xpath_text, -) +from ..utils import fix_xml_ampersands, xpath_text class DataTruncatedError(Exception): diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py index 217b89e3f..a2a2fe950 100644 --- a/yt_dlp/downloader/fragment.py +++ b/yt_dlp/downloader/fragment.py @@ -13,15 +13,11 @@ except ImportError: from .common import FileDownloader from .http import HttpFD from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7 -from ..compat import ( - compat_os_name, - compat_urllib_error, - compat_struct_pack, -) +from ..compat import compat_os_name, compat_struct_pack, compat_urllib_error from ..utils import ( DownloadError, - error_to_compat_str, encodeFilename, + error_to_compat_str, sanitized_Request, traverse_obj, ) diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py index 00695f93f..2d65f48ae 100644 --- a/yt_dlp/downloader/hls.py +++ b/yt_dlp/downloader/hls.py @@ -1,21 +1,13 @@ -import re -import io import binascii +import io +import re -from ..downloader import get_suitable_downloader -from .fragment import FragmentFD from .external import FFmpegFD - -from ..compat import ( - compat_pycrypto_AES, - compat_urlparse, -) -from ..utils import ( - parse_m3u8_attributes, - update_url_query, - bug_reports_message, -) +from .fragment import FragmentFD from .. import webvtt +from ..compat import compat_pycrypto_AES, compat_urlparse +from ..downloader import get_suitable_downloader +from ..utils import bug_reports_message, parse_m3u8_attributes, update_url_query class HlsFD(FragmentFD): diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py index 03efbf1cd..d590dbfbd 100644 --- a/yt_dlp/downloader/http.py +++ b/yt_dlp/downloader/http.py @@ -1,24 +1,21 @@ import os +import random import ssl import time -import random from .common import FileDownloader -from ..compat import ( - compat_urllib_error, - compat_http_client -) +from ..compat import compat_http_client, compat_urllib_error from ..utils import ( ContentTooShortError, + ThrottledDownload, + XAttrMetadataError, + XAttrUnavailableError, encodeFilename, int_or_none, parse_http_range, sanitized_Request, - ThrottledDownload, try_call, write_xattr, - XAttrMetadataError, - XAttrUnavailableError, ) RESPONSE_READ_EXCEPTIONS = (TimeoutError, ConnectionError, ssl.SSLError, compat_http_client.HTTPException) diff --git a/yt_dlp/downloader/ism.py b/yt_dlp/downloader/ism.py index ca4ca3a19..82ed51e88 100644 --- a/yt_dlp/downloader/ism.py +++ b/yt_dlp/downloader/ism.py @@ -1,13 +1,10 @@ -import time import binascii import io import struct +import time from .fragment import FragmentFD -from ..compat import ( - compat_urllib_error, -) - +from ..compat import compat_urllib_error u8 = struct.Struct('>B') u88 = struct.Struct('>Bx') diff --git a/yt_dlp/downloader/mhtml.py b/yt_dlp/downloader/mhtml.py index 5a322f1db..7bc3ab049 100644 --- a/yt_dlp/downloader/mhtml.py +++ b/yt_dlp/downloader/mhtml.py @@ -4,12 +4,7 @@ import re import uuid from .fragment import FragmentFD -from ..utils import ( - escapeHTML, - formatSeconds, - srt_subtitles_timecode, - urljoin, -) +from ..utils import escapeHTML, formatSeconds, srt_subtitles_timecode, urljoin from ..version import __version__ as YT_DLP_VERSION diff --git a/yt_dlp/downloader/rtmp.py b/yt_dlp/downloader/rtmp.py index 12aa04cf3..3464eeef9 100644 --- a/yt_dlp/downloader/rtmp.py +++ b/yt_dlp/downloader/rtmp.py @@ -6,11 +6,11 @@ import time from .common import FileDownloader from ..compat import compat_str from ..utils import ( + Popen, check_executable, - encodeFilename, encodeArgument, + encodeFilename, get_exe_version, - Popen, ) diff --git a/yt_dlp/downloader/rtsp.py b/yt_dlp/downloader/rtsp.py index 26dbd9ef7..e89269fed 100644 --- a/yt_dlp/downloader/rtsp.py +++ b/yt_dlp/downloader/rtsp.py @@ -2,10 +2,7 @@ import os import subprocess from .common import FileDownloader -from ..utils import ( - check_executable, - encodeFilename, -) +from ..utils import check_executable, encodeFilename class RtspFD(FileDownloader): diff --git a/yt_dlp/downloader/websocket.py b/yt_dlp/downloader/websocket.py index daac34884..96d113846 100644 --- a/yt_dlp/downloader/websocket.py +++ b/yt_dlp/downloader/websocket.py @@ -1,6 +1,6 @@ +import asyncio import os import signal -import asyncio import threading try: diff --git a/yt_dlp/downloader/youtube_live_chat.py b/yt_dlp/downloader/youtube_live_chat.py index 36c82b03b..7f06dfb48 100644 --- a/yt_dlp/downloader/youtube_live_chat.py +++ b/yt_dlp/downloader/youtube_live_chat.py @@ -3,13 +3,8 @@ import time from .fragment import FragmentFD from ..compat import compat_urllib_error -from ..utils import ( - try_get, - dict_get, - int_or_none, - RegexNotFoundError, -) from ..extractor.youtube import YoutubeBaseInfoExtractor as YT_BaseIE +from ..utils import RegexNotFoundError, dict_get, int_or_none, try_get class YoutubeLiveChatFD(FragmentFD): diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py index c7db05475..0dc8dea26 100644 --- a/yt_dlp/extractor/abematv.py +++ b/yt_dlp/extractor/abematv.py @@ -1,35 +1,31 @@ -import io -import json -import time +import base64 +import binascii import hashlib import hmac +import io +import json import re import struct +import time import urllib.response import uuid -from base64 import urlsafe_b64encode -from binascii import unhexlify from .common import InfoExtractor from ..aes import aes_ecb_decrypt -from ..compat import ( - compat_urllib_parse_urlparse, - compat_urllib_request, -) +from ..compat import compat_urllib_parse_urlparse, compat_urllib_request from ..utils import ( ExtractorError, + bytes_to_intlist, decode_base, int_or_none, + intlist_to_bytes, request_to_url, time_seconds, - update_url_query, traverse_obj, - intlist_to_bytes, - bytes_to_intlist, + update_url_query, urljoin, ) - # NOTE: network handler related code is temporary thing until network stack overhaul PRs are merged (#2861/#2862) def add_opener(ydl, handler): @@ -130,7 +126,7 @@ class AbemaLicenseHandler(compat_urllib_request.BaseHandler): encvideokey = bytes_to_intlist(struct.pack('>QQ', res >> 64, res & 0xffffffffffffffff)) h = hmac.new( - unhexlify(self.HKEY), + binascii.unhexlify(self.HKEY), (license_response['cid'] + self.ie._DEVICE_ID).encode('utf-8'), digestmod=hashlib.sha256) enckey = bytes_to_intlist(h.digest()) @@ -238,7 +234,7 @@ class AbemaTVIE(AbemaTVBaseIE): def mix_twist(nonce): nonlocal tmp - mix_once(urlsafe_b64encode(tmp).rstrip(b'=') + nonce) + mix_once(base64.urlsafe_b64encode(tmp).rstrip(b'=') + nonce) mix_once(self._SECRETKEY) mix_tmp(time_struct.tm_mon) @@ -247,7 +243,7 @@ class AbemaTVIE(AbemaTVBaseIE): mix_twist(ts_1hour_str) mix_tmp(time_struct.tm_hour % 5) - return urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8') + return base64.urlsafe_b64encode(tmp).rstrip(b'=').decode('utf-8') def _get_device_token(self): if self._USERTOKEN: diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py index ef22c7876..10b297708 100644 --- a/yt_dlp/extractor/common.py +++ b/yt_dlp/extractor/common.py @@ -1,16 +1,16 @@ import base64 import collections -import xml.etree.ElementTree import hashlib import itertools import json +import math import netrc import os import random import re import sys import time -import math +import xml.etree.ElementTree from ..compat import ( compat_cookiejar_Cookie, @@ -29,11 +29,15 @@ from ..compat import ( compat_urlparse, ) from ..downloader import FileDownloader -from ..downloader.f4m import ( - get_base_url, - remove_encrypted_media, -) +from ..downloader.f4m import get_base_url, remove_encrypted_media from ..utils import ( + JSON_LD_RE, + NO_DEFAULT, + ExtractorError, + GeoRestrictedError, + GeoUtils, + RegexNotFoundError, + UnsupportedError, age_restricted, base_url, bug_reports_message, @@ -44,20 +48,15 @@ from ..utils import ( encode_data_uri, error_to_compat_str, extract_attributes, - ExtractorError, filter_dict, fix_xml_ampersands, float_or_none, format_field, - GeoRestrictedError, - GeoUtils, int_or_none, join_nonempty, js_to_json, - JSON_LD_RE, mimetype2ext, network_exceptions, - NO_DEFAULT, orderedSet, parse_bitrate, parse_codecs, @@ -65,7 +64,6 @@ from ..utils import ( parse_iso8601, parse_m3u8_attributes, parse_resolution, - RegexNotFoundError, sanitize_filename, sanitized_Request, str_or_none, @@ -74,7 +72,6 @@ from ..utils import ( traverse_obj, try_get, unescapeHTML, - UnsupportedError, unified_strdate, unified_timestamp, update_Request, diff --git a/yt_dlp/extractor/commonprotocols.py b/yt_dlp/extractor/commonprotocols.py index 40475f7ec..e8f19b9e0 100644 --- a/yt_dlp/extractor/commonprotocols.py +++ b/yt_dlp/extractor/commonprotocols.py @@ -1,7 +1,5 @@ from .common import InfoExtractor -from ..compat import ( - compat_urlparse, -) +from ..compat import compat_urlparse class RtmpIE(InfoExtractor): diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py index f44f19a54..c708b4cee 100644 --- a/yt_dlp/extractor/generic.py +++ b/yt_dlp/extractor/generic.py @@ -2,8 +2,107 @@ import os import re import xml.etree.ElementTree +from .ant1newsgr import Ant1NewsGrEmbedIE +from .anvato import AnvatoIE +from .apa import APAIE +from .arcpublishing import ArcPublishingIE +from .arkena import ArkenaIE +from .arte import ArteTVEmbedIE +from .bitchute import BitChuteIE +from .blogger import BloggerIE +from .brightcove import BrightcoveLegacyIE, BrightcoveNewIE +from .channel9 import Channel9IE +from .cloudflarestream import CloudflareStreamIE from .common import InfoExtractor +from .commonprotocols import RtmpIE +from .condenast import CondeNastIE +from .dailymail import DailyMailIE +from .dailymotion import DailymotionIE +from .dbtv import DBTVIE +from .digiteka import DigitekaIE +from .drtuber import DrTuberIE +from .eagleplatform import EaglePlatformIE +from .ertgr import ERTWebtvEmbedIE +from .expressen import ExpressenIE +from .facebook import FacebookIE +from .foxnews import FoxNewsIE +from .gedidigital import GediDigitalIE +from .gfycat import GfycatIE +from .glomex import GlomexEmbedIE +from .googledrive import GoogleDriveIE +from .indavideo import IndavideoEmbedIE +from .instagram import InstagramIE +from .joj import JojIE +from .jwplatform import JWPlatformIE +from .kaltura import KalturaIE +from .kinja import KinjaEmbedIE +from .limelight import LimelightBaseIE +from .mainstreaming import MainStreamingIE +from .medialaan import MedialaanIE +from .mediaset import MediasetIE +from .mediasite import MediasiteIE +from .megaphone import MegaphoneIE +from .megatvcom import MegaTVComEmbedIE +from .mofosex import MofosexEmbedIE +from .mtv import MTVServicesEmbeddedIE +from .myvi import MyviIE +from .nbc import NBCSportsVPlayerIE +from .nexx import NexxEmbedIE, NexxIE +from .odnoklassniki import OdnoklassnikiIE +from .onionstudios import OnionStudiosIE +from .ooyala import OoyalaIE +from .panopto import PanoptoBaseIE +from .peertube import PeerTubeIE +from .piksel import PikselIE +from .pladform import PladformIE +from .pornhub import PornHubIE +from .rcs import RCSEmbedsIE +from .redtube import RedTubeIE +from .rumble import RumbleEmbedIE +from .rutube import RutubeIE +from .rutv import RUTVIE +from .ruutu import RuutuIE +from .senategov import SenateISVPIE +from .simplecast import SimplecastIE +from .soundcloud import SoundcloudEmbedIE +from .spankwire import SpankwireIE +from .sportbox import SportBoxIE +from .springboardplatform import SpringboardPlatformIE +from .svt import SVTIE +from .teachable import TeachableIE +from .ted import TedEmbedIE +from .theplatform import ThePlatformIE +from .threeqsdn import ThreeQSDNIE +from .tnaflix import TNAFlixNetworkEmbedIE +from .tube8 import Tube8IE +from .tunein import TuneInBaseIE +from .tvc import TVCIE +from .tvopengr import TVOpenGrEmbedIE +from .tvp import TVPEmbedIE +from .twentymin import TwentyMinutenIE +from .udn import UDNEmbedIE +from .ustream import UstreamIE +from .vbox7 import Vbox7IE +from .vice import ViceIE +from .videa import VideaIE +from .videomore import VideomoreIE +from .videopress import VideoPressIE +from .viewlift import ViewLiftEmbedIE +from .vimeo import VHXEmbedIE, VimeoIE +from .viqeo import ViqeoIE +from .vk import VKIE +from .vshare import VShareIE +from .vzaar import VzaarIE +from .washingtonpost import WashingtonPostIE +from .webcaster import WebcasterFeedIE +from .wimtv import WimTVIE +from .wistia import WistiaIE +from .xfileshare import XFileShareIE +from .xhamster import XHamsterEmbedIE +from .yapfiles import YapFilesIE +from .youporn import YouPornIE from .youtube import YoutubeIE +from .zype import ZypeIE from ..compat import ( compat_etree_fromstring, compat_str, @@ -11,15 +110,16 @@ from ..compat import ( compat_urlparse, ) from ..utils import ( + KNOWN_EXTENSIONS, + ExtractorError, + HEADRequest, + UnsupportedError, determine_ext, dict_get, - ExtractorError, float_or_none, - HEADRequest, int_or_none, is_html, js_to_json, - KNOWN_EXTENSIONS, merge_dicts, mimetype2ext, orderedSet, @@ -31,120 +131,11 @@ from ..utils import ( unescapeHTML, unified_timestamp, unsmuggle_url, - UnsupportedError, url_or_none, xpath_attr, xpath_text, xpath_with_ns, ) -from .commonprotocols import RtmpIE -from .brightcove import ( - BrightcoveLegacyIE, - BrightcoveNewIE, -) -from .nexx import ( - NexxIE, - NexxEmbedIE, -) -from .nbc import NBCSportsVPlayerIE -from .ooyala import OoyalaIE -from .rutv import RUTVIE -from .tvc import TVCIE -from .sportbox import SportBoxIE -from .myvi import MyviIE -from .condenast import CondeNastIE -from .udn import UDNEmbedIE -from .senategov import SenateISVPIE -from .svt import SVTIE -from .pornhub import PornHubIE -from .xhamster import XHamsterEmbedIE -from .tnaflix import TNAFlixNetworkEmbedIE -from .drtuber import DrTuberIE -from .redtube import RedTubeIE -from .tube8 import Tube8IE -from .mofosex import MofosexEmbedIE -from .spankwire import SpankwireIE -from .youporn import YouPornIE -from .vimeo import ( - VimeoIE, - VHXEmbedIE, -) -from .dailymotion import DailymotionIE -from .dailymail import DailyMailIE -from .onionstudios import OnionStudiosIE -from .viewlift import ViewLiftEmbedIE -from .mtv import MTVServicesEmbeddedIE -from .pladform import PladformIE -from .videomore import VideomoreIE -from .webcaster import WebcasterFeedIE -from .googledrive import GoogleDriveIE -from .jwplatform import JWPlatformIE -from .digiteka import DigitekaIE -from .arkena import ArkenaIE -from .instagram import InstagramIE -from .threeqsdn import ThreeQSDNIE -from .theplatform import ThePlatformIE -from .kaltura import KalturaIE -from .eagleplatform import EaglePlatformIE -from .facebook import FacebookIE -from .soundcloud import SoundcloudEmbedIE -from .tunein import TuneInBaseIE -from .vbox7 import Vbox7IE -from .dbtv import DBTVIE -from .piksel import PikselIE -from .videa import VideaIE -from .twentymin import TwentyMinutenIE -from .ustream import UstreamIE -from .arte import ArteTVEmbedIE -from .videopress import VideoPressIE -from .rutube import RutubeIE -from .glomex import GlomexEmbedIE -from .megatvcom import MegaTVComEmbedIE -from .ant1newsgr import Ant1NewsGrEmbedIE -from .limelight import LimelightBaseIE -from .anvato import AnvatoIE -from .washingtonpost import WashingtonPostIE -from .wistia import WistiaIE -from .mediaset import MediasetIE -from .joj import JojIE -from .megaphone import MegaphoneIE -from .vzaar import VzaarIE -from .channel9 import Channel9IE -from .vshare import VShareIE -from .mediasite import MediasiteIE -from .springboardplatform import SpringboardPlatformIE -from .ted import TedEmbedIE -from .yapfiles import YapFilesIE -from .vice import ViceIE -from .xfileshare import XFileShareIE -from .cloudflarestream import CloudflareStreamIE -from .peertube import PeerTubeIE -from .teachable import TeachableIE -from .indavideo import IndavideoEmbedIE -from .apa import APAIE -from .foxnews import FoxNewsIE -from .viqeo import ViqeoIE -from .expressen import ExpressenIE -from .zype import ZypeIE -from .odnoklassniki import OdnoklassnikiIE -from .vk import VKIE -from .kinja import KinjaEmbedIE -from .gedidigital import GediDigitalIE -from .rcs import RCSEmbedsIE -from .bitchute import BitChuteIE -from .rumble import RumbleEmbedIE -from .arcpublishing import ArcPublishingIE -from .medialaan import MedialaanIE -from .simplecast import SimplecastIE -from .wimtv import WimTVIE -from .tvopengr import TVOpenGrEmbedIE -from .ertgr import ERTWebtvEmbedIE -from .tvp import TVPEmbedIE -from .blogger import BloggerIE -from .mainstreaming import MainStreamingIE -from .gfycat import GfycatIE -from .panopto import PanoptoBaseIE -from .ruutu import RuutuIE class GenericIE(InfoExtractor): diff --git a/yt_dlp/extractor/mtv.py b/yt_dlp/extractor/mtv.py index 3ef851e0b..d161c33c1 100644 --- a/yt_dlp/extractor/mtv.py +++ b/yt_dlp/extractor/mtv.py @@ -1,9 +1,7 @@ import re from .common import InfoExtractor -from ..compat import ( - compat_str, -) +from ..compat import compat_str from ..utils import ( ExtractorError, find_xpath_attr, diff --git a/yt_dlp/extractor/noz.py b/yt_dlp/extractor/noz.py index 22cb08e8a..b42a56f7e 100644 --- a/yt_dlp/extractor/noz.py +++ b/yt_dlp/extractor/noz.py @@ -1,13 +1,11 @@ from .common import InfoExtractor -from ..compat import ( - compat_urllib_parse_unquote, -) from ..utils import ( int_or_none, find_xpath_attr, xpath_text, update_url_query, ) +from ..compat import compat_urllib_parse_unquote class NozIE(InfoExtractor): diff --git a/yt_dlp/extractor/openload.py b/yt_dlp/extractor/openload.py index 41ef2e892..f2600aaa4 100644 --- a/yt_dlp/extractor/openload.py +++ b/yt_dlp/extractor/openload.py @@ -3,16 +3,14 @@ import os import subprocess import tempfile -from ..compat import ( - compat_urlparse, -) +from ..compat import compat_urlparse from ..utils import ( + ExtractorError, + Popen, check_executable, encodeArgument, - ExtractorError, get_exe_version, is_outdated_version, - Popen, ) diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 21c6143bd..dee1b2315 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -10,9 +10,9 @@ import os.path import random import re import sys +import threading import time import traceback -import threading from .common import InfoExtractor, SearchInfoExtractor from ..compat import ( @@ -27,12 +27,13 @@ from ..compat import ( ) from ..jsinterp import JSInterpreter from ..utils import ( + NO_DEFAULT, + ExtractorError, bug_reports_message, clean_html, datetime_from_str, dict_get, error_to_compat_str, - ExtractorError, float_or_none, format_field, get_first, @@ -42,7 +43,6 @@ from ..utils import ( js_to_json, mimetype2ext, network_exceptions, - NO_DEFAULT, orderedSet, parse_codecs, parse_count, @@ -68,7 +68,6 @@ from ..utils import ( variadic, ) - # any clients starting with _ cannot be explicity requested by the user INNERTUBE_CLIENTS = { 'web': { diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py index 3695a282d..001836887 100644 --- a/yt_dlp/jsinterp.py +++ b/yt_dlp/jsinterp.py @@ -1,12 +1,9 @@ -from collections.abc import MutableMapping import json import operator import re +from collections.abc import MutableMapping -from .utils import ( - ExtractorError, - remove_quotes, -) +from .utils import ExtractorError, remove_quotes _OPERATORS = [ ('|', operator.or_), diff --git a/yt_dlp/minicurses.py b/yt_dlp/minicurses.py index d7a8ffddd..9fd679a48 100644 --- a/yt_dlp/minicurses.py +++ b/yt_dlp/minicurses.py @@ -1,7 +1,7 @@ import functools from threading import Lock -from .utils import supports_terminal_sequences, write_string +from .utils import supports_terminal_sequences, write_string CONTROL_SEQUENCES = { 'DOWN': '\n', diff --git a/yt_dlp/options.py b/yt_dlp/options.py index c434e32b9..243beab4d 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -1,26 +1,11 @@ -import os.path import optparse +import os.path import re import shlex import sys -from .compat import ( - compat_expanduser, - compat_get_terminal_size, - compat_getenv, -) -from .utils import ( - Config, - expand_path, - get_executable_path, - OUTTMPL_TYPES, - POSTPROCESS_WHEN, - remove_end, - write_string, -) +from .compat import compat_expanduser, compat_get_terminal_size, compat_getenv from .cookies import SUPPORTED_BROWSERS, SUPPORTED_KEYRINGS -from .version import __version__ - from .downloader.external import list_external_downloaders from .postprocessor import ( FFmpegExtractAudioPP, @@ -30,6 +15,16 @@ from .postprocessor import ( SponsorBlockPP, ) from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE +from .utils import ( + OUTTMPL_TYPES, + POSTPROCESS_WHEN, + Config, + expand_path, + get_executable_path, + remove_end, + write_string, +) +from .version import __version__ def parseOpts(overrideArguments=None, ignore_config_files='if_override'): diff --git a/yt_dlp/postprocessor/__init__.py b/yt_dlp/postprocessor/__init__.py index e47631eb6..f168be46a 100644 --- a/yt_dlp/postprocessor/__init__.py +++ b/yt_dlp/postprocessor/__init__.py @@ -1,27 +1,25 @@ # flake8: noqa: F401 -from ..utils import load_plugins - from .common import PostProcessor from .embedthumbnail import EmbedThumbnailPP -from .exec import ExecPP, ExecAfterDownloadPP +from .exec import ExecAfterDownloadPP, ExecPP from .ffmpeg import ( - FFmpegPostProcessor, - FFmpegCopyStreamPP, FFmpegConcatPP, + FFmpegCopyStreamPP, FFmpegEmbedSubtitlePP, FFmpegExtractAudioPP, FFmpegFixupDuplicateMoovPP, FFmpegFixupDurationPP, - FFmpegFixupStretchedPP, - FFmpegFixupTimestampPP, FFmpegFixupM3u8PP, FFmpegFixupM4aPP, + FFmpegFixupStretchedPP, + FFmpegFixupTimestampPP, FFmpegMergerPP, FFmpegMetadataPP, + FFmpegPostProcessor, + FFmpegSplitChaptersPP, FFmpegSubtitlesConvertorPP, FFmpegThumbnailsConvertorPP, - FFmpegSplitChaptersPP, FFmpegVideoConvertorPP, FFmpegVideoRemuxerPP, ) @@ -35,6 +33,7 @@ from .movefilesafterdownload import MoveFilesAfterDownloadPP from .sponskrub import SponSkrubPP from .sponsorblock import SponsorBlockPP from .xattrpp import XAttrMetadataPP +from ..utils import load_plugins _PLUGIN_CLASSES = load_plugins('postprocessor', 'PP', globals()) diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py index 3f55b24f2..ce6dec2f5 100644 --- a/yt_dlp/postprocessor/common.py +++ b/yt_dlp/postprocessor/common.py @@ -6,10 +6,10 @@ import time import urllib.error from ..utils import ( + PostProcessingError, _configuration_args, encodeFilename, network_exceptions, - PostProcessingError, sanitized_Request, write_string, ) diff --git a/yt_dlp/postprocessor/embedthumbnail.py b/yt_dlp/postprocessor/embedthumbnail.py index 2fca97784..5469f25e0 100644 --- a/yt_dlp/postprocessor/embedthumbnail.py +++ b/yt_dlp/postprocessor/embedthumbnail.py @@ -1,11 +1,11 @@ import base64 import imghdr import os -import subprocess import re +import subprocess try: - from mutagen.flac import Picture, FLAC + from mutagen.flac import FLAC, Picture from mutagen.mp4 import MP4, MP4Cover from mutagen.oggopus import OggOpus from mutagen.oggvorbis import OggVorbis @@ -14,17 +14,14 @@ except ImportError: has_mutagen = False from .common import PostProcessor -from .ffmpeg import ( - FFmpegPostProcessor, - FFmpegThumbnailsConvertorPP, -) +from .ffmpeg import FFmpegPostProcessor, FFmpegThumbnailsConvertorPP from ..utils import ( + Popen, + PostProcessingError, check_executable, encodeArgument, encodeFilename, error_to_compat_str, - Popen, - PostProcessingError, prepend_extension, shell_quote, ) diff --git a/yt_dlp/postprocessor/exec.py b/yt_dlp/postprocessor/exec.py index 6621889d5..cfc83167c 100644 --- a/yt_dlp/postprocessor/exec.py +++ b/yt_dlp/postprocessor/exec.py @@ -2,11 +2,7 @@ import subprocess from .common import PostProcessor from ..compat import compat_shlex_quote -from ..utils import ( - encodeArgument, - PostProcessingError, - variadic, -) +from ..utils import PostProcessingError, encodeArgument, variadic class ExecPP(PostProcessor): diff --git a/yt_dlp/postprocessor/ffmpeg.py b/yt_dlp/postprocessor/ffmpeg.py index 3175c8d10..69182618b 100644 --- a/yt_dlp/postprocessor/ffmpeg.py +++ b/yt_dlp/postprocessor/ffmpeg.py @@ -1,27 +1,26 @@ import collections import itertools +import json import os +import re import subprocess import time -import re -import json from .common import AudioConversionError, PostProcessor - from ..compat import compat_str from ..utils import ( + ISO639Utils, + Popen, + PostProcessingError, + _get_exe_version_output, + detect_exe_version, determine_ext, dfxp2srt, encodeArgument, encodeFilename, float_or_none, - _get_exe_version_output, - detect_exe_version, is_outdated_version, - ISO639Utils, orderedSet, - Popen, - PostProcessingError, prepend_extension, replace_extension, shell_quote, @@ -30,7 +29,6 @@ from ..utils import ( write_json_file, ) - EXT_TO_OUT_FORMATS = { 'aac': 'adts', 'flac': 'flac', diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py index 22506bc21..7e2c23288 100644 --- a/yt_dlp/postprocessor/modify_chapters.py +++ b/yt_dlp/postprocessor/modify_chapters.py @@ -3,17 +3,9 @@ import heapq import os from .common import PostProcessor -from .ffmpeg import ( - FFmpegPostProcessor, - FFmpegSubtitlesConvertorPP -) +from .ffmpeg import FFmpegPostProcessor, FFmpegSubtitlesConvertorPP from .sponsorblock import SponsorBlockPP -from ..utils import ( - orderedSet, - PostProcessingError, - prepend_extension, -) - +from ..utils import PostProcessingError, orderedSet, prepend_extension _TINY_CHAPTER_DURATION = 1 DEFAULT_SPONSORBLOCK_CHAPTER_TITLE = '[SponsorBlock]: %(category_names)l' diff --git a/yt_dlp/postprocessor/movefilesafterdownload.py b/yt_dlp/postprocessor/movefilesafterdownload.py index bc3d15ca4..436d13227 100644 --- a/yt_dlp/postprocessor/movefilesafterdownload.py +++ b/yt_dlp/postprocessor/movefilesafterdownload.py @@ -3,10 +3,10 @@ import shutil from .common import PostProcessor from ..utils import ( + PostProcessingError, decodeFilename, encodeFilename, make_dir, - PostProcessingError, ) diff --git a/yt_dlp/postprocessor/sponskrub.py b/yt_dlp/postprocessor/sponskrub.py index 38089de08..1a9f5dc66 100644 --- a/yt_dlp/postprocessor/sponskrub.py +++ b/yt_dlp/postprocessor/sponskrub.py @@ -4,15 +4,15 @@ import subprocess from .common import PostProcessor from ..utils import ( + Popen, + PostProcessingError, check_executable, cli_option, encodeArgument, encodeFilename, + prepend_extension, shell_quote, str_or_none, - Popen, - PostProcessingError, - prepend_extension, ) diff --git a/yt_dlp/postprocessor/sponsorblock.py b/yt_dlp/postprocessor/sponsorblock.py index 7943014e2..501e30320 100644 --- a/yt_dlp/postprocessor/sponsorblock.py +++ b/yt_dlp/postprocessor/sponsorblock.py @@ -1,6 +1,6 @@ -from hashlib import sha256 import json import re +from hashlib import sha256 from .ffmpeg import FFmpegPostProcessor from ..compat import compat_urllib_parse_urlencode diff --git a/yt_dlp/postprocessor/xattrpp.py b/yt_dlp/postprocessor/xattrpp.py index 5ad8509e7..3c431941b 100644 --- a/yt_dlp/postprocessor/xattrpp.py +++ b/yt_dlp/postprocessor/xattrpp.py @@ -1,11 +1,11 @@ from .common import PostProcessor from ..compat import compat_os_name from ..utils import ( - hyphenate_date, - write_xattr, PostProcessingError, XAttrMetadataError, XAttrUnavailableError, + hyphenate_date, + write_xattr, ) diff --git a/yt_dlp/socks.py b/yt_dlp/socks.py index ffa960e03..56fab08ab 100644 --- a/yt_dlp/socks.py +++ b/yt_dlp/socks.py @@ -9,11 +9,7 @@ import collections import socket -from .compat import ( - compat_ord, - compat_struct_pack, - compat_struct_unpack, -) +from .compat import compat_ord, compat_struct_pack, compat_struct_unpack __author__ = 'Timo Schmid <coding@timoschmid.de>' diff --git a/yt_dlp/update.py b/yt_dlp/update.py index 7db260e96..eea08ce43 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -8,8 +8,7 @@ import traceback from zipimport import zipimporter from .compat import compat_realpath -from .utils import encode_compat_str, Popen, write_string - +from .utils import Popen, encode_compat_str, write_string from .version import __version__ diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 91e1a9870..25ac864f3 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -9,8 +9,8 @@ import collections import contextlib import ctypes import datetime -import email.utils import email.header +import email.utils import errno import functools import gzip @@ -22,11 +22,13 @@ import itertools import json import locale import math +import mimetypes import operator import os import platform import random import re +import shlex import socket import ssl import subprocess @@ -34,16 +36,11 @@ import sys import tempfile import time import traceback +import urllib.parse import xml.etree.ElementTree import zlib -import mimetypes -import urllib.parse -import shlex from .compat import ( - compat_HTMLParseError, - compat_HTMLParser, - compat_HTTPError, compat_brotli, compat_chr, compat_cookiejar, @@ -51,7 +48,10 @@ from .compat import ( compat_expanduser, compat_html_entities, compat_html_entities_html5, + compat_HTMLParseError, + compat_HTMLParser, compat_http_client, + compat_HTTPError, compat_os_name, compat_parse_qs, compat_shlex_quote, @@ -59,18 +59,14 @@ from .compat import ( compat_struct_pack, compat_struct_unpack, compat_urllib_error, + compat_urllib_parse_unquote_plus, compat_urllib_parse_urlencode, compat_urllib_parse_urlparse, - compat_urllib_parse_unquote_plus, compat_urllib_request, compat_urlparse, compat_websockets, ) - -from .socks import ( - ProxyType, - sockssocket, -) +from .socks import ProxyType, sockssocket try: import certifi diff --git a/yt_dlp/webvtt.py b/yt_dlp/webvtt.py index e0d7f6743..3180eafde 100644 --- a/yt_dlp/webvtt.py +++ b/yt_dlp/webvtt.py @@ -8,13 +8,11 @@ Regular expressions based on the W3C WebVTT specification in RFC 8216 ยง3.5 <https://tools.ietf.org/html/rfc8216#section-3.5>. """ -import re import io +import re + +from .compat import compat_Match, compat_Pattern from .utils import int_or_none, timetuple_from_msec -from .compat import ( - compat_Pattern, - compat_Match, -) class _MatchParser: |