diff options
Diffstat (limited to 'devscripts')
| -rw-r--r-- | devscripts/bash-completion.in | 4 | ||||
| -rwxr-xr-x | devscripts/bash-completion.py | 8 | ||||
| -rw-r--r-- | devscripts/buildserver.py | 6 | ||||
| -rw-r--r-- | devscripts/check-porn.py | 6 | ||||
| -rw-r--r-- | devscripts/fish-completion.in | 2 | ||||
| -rwxr-xr-x | devscripts/fish-completion.py | 13 | ||||
| -rw-r--r-- | devscripts/generate_aes_testdata.py | 5 | ||||
| -rw-r--r-- | devscripts/lazy_load_template.py | 26 | ||||
| -rw-r--r-- | devscripts/logo.ico | bin | 0 -> 41043 bytes | |||
| -rwxr-xr-x | devscripts/make_contributing.py | 7 | ||||
| -rw-r--r-- | devscripts/make_lazy_extractors.py | 34 | ||||
| -rwxr-xr-x | devscripts/make_readme.py | 11 | ||||
| -rw-r--r-- | devscripts/make_supportedsites.py | 8 | ||||
| -rw-r--r-- | devscripts/prepare_manpage.py | 7 | ||||
| -rw-r--r-- | devscripts/run_tests.bat | 21 | ||||
| -rwxr-xr-x | devscripts/run_tests.sh | 36 | ||||
| -rw-r--r-- | devscripts/zsh-completion.in | 8 | ||||
| -rwxr-xr-x | devscripts/zsh-completion.py | 8 |
18 files changed, 119 insertions, 91 deletions
diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index 28bd23727..21f52798e 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -1,4 +1,4 @@ -__youtube_dl() +__yt_dlp() { local cur prev opts fileopts diropts keywords COMPREPLY=() @@ -26,4 +26,4 @@ __youtube_dl() fi } -complete -F __youtube_dl youtube-dl +complete -F __yt_dlp yt-dlp diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 3d1391334..46b4b2ff5 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals import os @@ -6,9 +6,9 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +import yt_dlp -BASH_COMPLETION_FILE = "youtube-dl.bash-completion" +BASH_COMPLETION_FILE = "completions/bash/yt-dlp" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -26,5 +26,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = yt_dlp.parseOpts()[0] build_completion(parser) diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py index 4a4295ba9..cd544b816 100644 --- a/devscripts/buildserver.py +++ b/devscripts/buildserver.py @@ -1,3 +1,5 @@ +# UNUSED + #!/usr/bin/python3 import argparse @@ -12,7 +14,7 @@ import traceback import os.path sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) -from youtube_dl.compat import ( +from yt_dlp.compat import ( compat_input, compat_http_server, compat_str, @@ -325,7 +327,7 @@ class YoutubeDLBuilder(object): authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] def __init__(self, **kwargs): - if self.repoName != 'youtube-dl': + if self.repoName != 'yt-dlp': raise BuildError('Invalid repository "%s"' % self.repoName) if self.user not in self.authorizedUsers: raise HTTPError('Unauthorized user "%s"' % self.user, 401) diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 740f04de0..50f6bebc6 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals """ @@ -15,8 +15,8 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import gettestcases -from youtube_dl.utils import compat_urllib_parse_urlparse -from youtube_dl.utils import compat_urllib_request +from yt_dlp.utils import compat_urllib_parse_urlparse +from yt_dlp.utils import compat_urllib_request if len(sys.argv) > 1: METHOD = 'LIST' diff --git a/devscripts/fish-completion.in b/devscripts/fish-completion.in index eb79765da..32938fbb4 100644 --- a/devscripts/fish-completion.in +++ b/devscripts/fish-completion.in @@ -2,4 +2,4 @@ {{commands}} -complete --command youtube-dl --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" +complete --command yt-dlp --arguments ":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 51d19dd33..fb45e0280 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals import optparse @@ -7,13 +7,14 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl -from youtube_dl.utils import shell_quote +import yt_dlp +from yt_dlp.utils import shell_quote -FISH_COMPLETION_FILE = 'youtube-dl.fish' +FISH_COMPLETION_FILE = 'completions/fish/yt-dlp.fish' FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' EXTRA_ARGS = { + 'remux-video': ['--arguments', 'mp4 mkv', '--exclusive'], 'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'], # Options that need a file parameter @@ -30,7 +31,7 @@ def build_completion(opt_parser): for group in opt_parser.option_groups: for option in group.option_list: long_option = option.get_opt_string().strip('-') - complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option] + complete_cmd = ['complete', '--command', 'yt-dlp', '--long-option', long_option] if option._short_opts: complete_cmd += ['--short-option', option._short_opts[0].strip('-')] if option.help != optparse.SUPPRESS_HELP: @@ -45,5 +46,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = youtube_dl.parseOpts()[0] +parser = yt_dlp.parseOpts()[0] build_completion(parser) diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index e3df42cc2..0979eee5b 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from __future__ import unicode_literals import codecs @@ -7,8 +8,8 @@ import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.utils import intlist_to_bytes -from youtube_dl.aes import aes_encrypt, key_expansion +from yt_dlp.utils import intlist_to_bytes +from yt_dlp.aes import aes_encrypt, key_expansion secret_msg = b'Secret message goes here' diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index c4e5fc1f4..da89e070d 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -1,19 +1,31 @@ # coding: utf-8 -from __future__ import unicode_literals - import re +from ..utils import bug_reports_message, write_string + + +class LazyLoadMetaClass(type): + def __getattr__(cls, name): + if '_real_class' not in cls.__dict__: + write_string( + f'WARNING: Falling back to normal extractor since lazy extractor ' + f'{cls.__name__} does not have attribute {name}{bug_reports_message()}') + return getattr(cls._get_real_class(), name) + -class LazyLoadExtractor(object): +class LazyLoadExtractor(metaclass=LazyLoadMetaClass): _module = None + _WORKING = True @classmethod - def ie_key(cls): - return cls.__name__[:-2] + def _get_real_class(cls): + if '_real_class' not in cls.__dict__: + mod = __import__(cls._module, fromlist=(cls.__name__,)) + cls._real_class = getattr(mod, cls.__name__) + return cls._real_class def __new__(cls, *args, **kwargs): - mod = __import__(cls._module, fromlist=(cls.__name__,)) - real_cls = getattr(mod, cls.__name__) + real_cls = cls._get_real_class() instance = real_cls.__new__(real_cls) instance.__init__(*args, **kwargs) return instance diff --git a/devscripts/logo.ico b/devscripts/logo.ico Binary files differnew file mode 100644 index 000000000..5503a4350 --- /dev/null +++ b/devscripts/logo.ico diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 226d1a5d6..6b1b8219c 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals import io @@ -7,6 +7,8 @@ import re def main(): + return # This is unused in yt-dlp + parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') options, args = parser.parse_args() if len(args) != 2: @@ -20,8 +22,7 @@ def main(): bug_text = re.search( r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) dev_text = re.search( - r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL', - readme).group(1) + r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING yt-dlp', readme).group(1) out = bug_text + dev_text diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 878ae72b1..427045b98 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from __future__ import unicode_literals, print_function from inspect import getsource @@ -6,27 +7,35 @@ import os from os.path import dirname as dirn import sys -print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) - sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) lazy_extractors_filename = sys.argv[1] if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) -from youtube_dl.extractor import _ALL_CLASSES -from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor +# Block plugins from loading +plugins_dirname = 'ytdlp_plugins' +plugins_blocked_dirname = 'ytdlp_plugins_blocked' +if os.path.exists(plugins_dirname): + os.rename(plugins_dirname, plugins_blocked_dirname) + +from yt_dlp.extractor import _ALL_CLASSES +from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor + +if os.path.exists(plugins_blocked_dirname): + os.rename(plugins_blocked_dirname, plugins_dirname) with open('devscripts/lazy_load_template.py', 'rt') as f: module_template = f.read() +CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] module_contents = [ - module_template + '\n' + getsource(InfoExtractor.suitable) + '\n', - 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] + module_template, + *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES], + '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] ie_template = ''' class {name}({bases}): - _VALID_URL = {valid_url!r} _module = '{module}' ''' @@ -47,14 +56,17 @@ def get_base_name(base): def build_lazy_ie(ie, name): - valid_url = getattr(ie, '_VALID_URL', None) s = ie_template.format( name=name, bases=', '.join(map(get_base_name, ie.__bases__)), - valid_url=valid_url, module=ie.__module__) + valid_url = getattr(ie, '_VALID_URL', None) + if valid_url: + s += f' _VALID_URL = {valid_url!r}\n' + if not ie._WORKING: + s += ' _WORKING = False\n' if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += '\n' + getsource(ie.suitable) + s += f'\n{getsource(ie.suitable)}' if hasattr(ie, '_make_valid_url'): # search extractors s += make_valid_template.format(valid_url=ie._make_valid_url()) @@ -92,7 +104,7 @@ for ie in ordered_cls: names.append(name) module_contents.append( - '_ALL_CLASSES = [{0}]'.format(', '.join(names))) + '\n_ALL_CLASSES = [{0}]'.format(', '.join(names))) module_src = '\n'.join(module_contents) + '\n' diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 8fbce0796..3f56af744 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,3 +1,8 @@ +#!/usr/bin/env python3 + +# yt-dlp --help | make_readme.py +# This must be run in a console of correct width + from __future__ import unicode_literals import io @@ -13,12 +18,12 @@ if isinstance(helptext, bytes): with io.open(README_FILE, encoding='utf-8') as f: oldreadme = f.read() -header = oldreadme[:oldreadme.index('# OPTIONS')] +header = oldreadme[:oldreadme.index('## General Options:')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] -options = helptext[helptext.index(' General Options:') + 19:] +options = helptext[helptext.index(' General Options:'):] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) -options = '# OPTIONS\n' + options + '\n' +options = options + '\n' with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 764795bc5..17a34843f 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals import io @@ -7,10 +7,10 @@ import os import sys -# Import youtube_dl +# Import yt_dlp ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') sys.path.insert(0, ROOT_DIR) -import youtube_dl +import yt_dlp def main(): @@ -33,7 +33,7 @@ def main(): ie_md += ' (Currently broken)' yield ie_md - ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()) + ies = sorted(yt_dlp.gen_extractors(), key=lambda i: i.IE_NAME.lower()) out = '# Supported sites\n' + ''.join( ' - ' + md + '\n' for md in gen_ies_md(ies)) diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 76bf873e1..485b39e9f 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 from __future__ import unicode_literals import io @@ -8,7 +9,7 @@ import re ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') -PREFIX = r'''%YOUTUBE-DL(1) +PREFIX = r'''%yt-dlp(1) # NAME @@ -16,7 +17,7 @@ youtube\-dl \- download videos from youtube.com or other video platforms # SYNOPSIS -**youtube-dl** \[OPTIONS\] URL [URL...] +**yt-dlp** \[OPTIONS\] URL [URL...] ''' @@ -33,7 +34,7 @@ def main(): readme = f.read() readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) - readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) + readme = re.sub(r'\s+yt-dlp \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) readme = PREFIX + readme readme = filter_options(readme) diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat index 01a79b6dd..b8bb393d9 100644 --- a/devscripts/run_tests.bat +++ b/devscripts/run_tests.bat @@ -1,17 +1,16 @@ +@setlocal @echo off +cd /d %~dp0.. -rem Keep this list in sync with the `offlinetest` target in Makefile -set DOWNLOAD_TESTS="age_restriction^|download^|socks^|subtitles^|write_annotations^|youtube_lists^|youtube_signature" - -if "%YTDL_TEST_SET%" == "core" ( - set test_set="-I test_("%DOWNLOAD_TESTS%")\.py" - set multiprocess_args="" -) else if "%YTDL_TEST_SET%" == "download" ( - set test_set="-I test_(?!"%DOWNLOAD_TESTS%").+\.py" - set multiprocess_args="--processes=4 --process-timeout=540" +if ["%~1"]==[""] ( + set "test_set="test"" +) else if ["%~1"]==["core"] ( + set "test_set="-m not download"" +) else if ["%~1"]==["download"] ( + set "test_set="-m "download"" ) else ( - echo YTDL_TEST_SET is not set or invalid + echo.Invalid test type "%~1". Use "core" ^| "download" exit /b 1 ) -nosetests test --verbose %test_set:"=% %multiprocess_args:"=% +pytest %test_set% diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index b8f48b9df..c9a75ba00 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,22 +1,14 @@ -#!/bin/bash - -# Keep this list in sync with the `offlinetest` target in Makefile -DOWNLOAD_TESTS="age_restriction|download|socks|subtitles|write_annotations|youtube_lists|youtube_signature" - -test_set="" -multiprocess_args="" - -case "$YTDL_TEST_SET" in - core) - test_set="-I test_($DOWNLOAD_TESTS)\.py" - ;; - download) - test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py" - multiprocess_args="--processes=4 --process-timeout=540" - ;; - *) - break - ;; -esac - -nosetests test --verbose $test_set $multiprocess_args +#!/bin/sh + +if [ -z $1 ]; then + test_set='test' +elif [ $1 = 'core' ]; then + test_set="-m not download" +elif [ $1 = 'download' ]; then + test_set="-m download" +else + echo 'Invalid test type "'$1'". Use "core" | "download"' + exit 1 +fi + +python3 -m pytest "$test_set" diff --git a/devscripts/zsh-completion.in b/devscripts/zsh-completion.in index b394a1ae7..9117d339e 100644 --- a/devscripts/zsh-completion.in +++ b/devscripts/zsh-completion.in @@ -1,6 +1,6 @@ -#compdef youtube-dl +#compdef yt-dlp -__youtube_dl() { +__yt_dlp() { local curcontext="$curcontext" fileopts diropts cur prev typeset -A opt_args fileopts="{{fileopts}}" @@ -16,6 +16,8 @@ __youtube_dl() { _path_files elif [[ ${prev} =~ ${diropts} ]]; then _path_files -/ + elif [[ ${prev} == "--remux-video" ]]; then + _arguments '*: :(mp4 mkv)' elif [[ ${prev} == "--recode-video" ]]; then _arguments '*: :(mp4 flv ogg webm mkv)' else @@ -25,4 +27,4 @@ __youtube_dl() { esac } -__youtube_dl
\ No newline at end of file +__yt_dlp
\ No newline at end of file diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 60aaf76cc..780df0de6 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import unicode_literals import os @@ -6,9 +6,9 @@ from os.path import dirname as dirn import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -import youtube_dl +import yt_dlp -ZSH_COMPLETION_FILE = "youtube-dl.zsh" +ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -45,5 +45,5 @@ def build_completion(opt_parser): f.write(template) -parser = youtube_dl.parseOpts()[0] +parser = yt_dlp.parseOpts()[0] build_completion(parser) |