From 5b836d47392d2ffb7205a30ac2b5786b208c3238 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Wed, 29 Jun 2022 11:25:40 +0100 Subject: [build] Consistent order for lazy extractors (#4220) Authored by: lamby --- devscripts/make_lazy_extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devscripts') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 785d66a6a..60fcc5ef0 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -94,7 +94,7 @@ def sort_ies(ies, ignored_bases): for c in classes[:]: bases = set(c.__bases__) - {object, *ignored_bases} restart = False - for b in bases: + for b in sorted(bases, key=lambda x: x.__name__): if b not in classes and b not in returned_classes: assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE' classes.insert(0, b) -- cgit v1.2.3 From ca9f1df25346816baacb13e875f3873c47be86e2 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 30 Jun 2022 04:06:27 +0530 Subject: [docs] Improve issue templates --- devscripts/make_issue_template.py | 47 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) (limited to 'devscripts') diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 54043ef4e..90e7e0b43 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import optparse +import re def read(fname): @@ -21,16 +22,56 @@ def read_version(fname): return locals()['__version__'] +VERBOSE_TMPL = ''' + - type: checkboxes + id: verbose + attributes: + label: Provide verbose output that clearly demonstrates the problem + options: + - label: Run **your** yt-dlp command with **-vU** flag added (`yt-dlp -vU `) + required: true + - label: Copy the WHOLE output (starting with `[debug] Command-line config`) and insert it below + required: true + - type: textarea + id: log + attributes: + label: Complete Verbose Output + description: | + It should start like this: + placeholder: | + [debug] Command-line config: ['-vU', 'test:youtube'] + [debug] Portable config "yt-dlp.conf": ['-i'] + [debug] Encodings: locale cp65001, fs utf-8, pref cp65001, out utf-8, error utf-8, screen utf-8 + [debug] yt-dlp version %(version)s [9d339c4] (win32_exe) + [debug] Python 3.8.10 (CPython 64bit) - Windows-10-10.0.22000-SP0 + [debug] Checking exe version: ffmpeg -bsfs + [debug] Checking exe version: ffprobe -bsfs + [debug] exe versions: ffmpeg N-106550-g072101bd52-20220410 (fdk,setts), ffprobe N-106624-g391ce570c8-20220415, phantomjs 2.1.1 + [debug] Optional libraries: Cryptodome-3.15.0, brotli-1.0.9, certifi-2022.06.15, mutagen-1.45.1, sqlite3-2.6.0, websockets-10.3 + [debug] Proxy map: {} + [debug] Fetching release info: https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest + Latest version: %(version)s, Current version: %(version)s + yt-dlp is up to date (%(version)s) + + render: shell + validations: + required: true +'''.strip() + + def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') - options, args = parser.parse_args() + _, args = parser.parse_args() if len(args) != 2: parser.error('Expected an input and an output filename') + fields = {'version': read_version('yt_dlp/version.py')} + fields['verbose'] = VERBOSE_TMPL % fields + fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose']) + infile, outfile = args with open(outfile, 'w', encoding='utf-8') as outf: - outf.write( - read(infile) % {'version': read_version('yt_dlp/version.py')}) + outf.write(read(infile) % fields) if __name__ == '__main__': -- cgit v1.2.3 From 5c0dc6e6035c4b92aa1a254ebb0284be75dd0d2b Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Fri, 1 Jul 2022 20:58:39 +0900 Subject: [devscripts/update-formulae] Do not change dependency section Closes #4223 --- devscripts/update-formulae.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devscripts') diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py index 02b869304..96b56b932 100644 --- a/devscripts/update-formulae.py +++ b/devscripts/update-formulae.py @@ -30,8 +30,8 @@ url = tarball_file['url'] with open(filename) as r: formulae_text = r.read() -formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text) -formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text) +formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1) +formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1) with open(filename, 'w') as w: w.write(formulae_text) -- cgit v1.2.3 From 8f97a15d1c7ebc10d0b51ce24632ac17b34a5f69 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 1 Aug 2022 06:52:03 +0530 Subject: [extractor] Framework for embed detection (#4307) --- devscripts/lazy_load_template.py | 6 ++++-- devscripts/make_lazy_extractors.py | 7 +------ 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index cdafaf1ef..a6e26b6f6 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -9,11 +9,13 @@ from ..utils import ( write_string, ) +# These bloat the lazy_extractors, so allow them to passthrough silently +ALLOWED_CLASSMETHODS = {'get_testcases', 'extract_from_webpage'} + class LazyLoadMetaClass(type): def __getattr__(cls, name): - # "_TESTS" bloat the lazy_extractors - if '_real_class' not in cls.__dict__ and name != 'get_testcases': + if '_real_class' not in cls.__dict__ and name not in ALLOWED_CLASSMETHODS: write_string( 'WARNING: Falling back to normal extractor since lazy extractor ' f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 60fcc5ef0..c9fdfb562 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -11,7 +11,7 @@ import optparse from inspect import getsource NO_ATTR = object() -STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit'] +STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_NETRC_MACHINE', 'age_limit'] CLASS_METHODS = [ 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable' ] @@ -116,11 +116,6 @@ def build_lazy_ie(ie, name, attr_base): }.get(base.__name__, base.__name__) for base in ie.__bases__) s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases) - valid_url = getattr(ie, '_VALID_URL', None) - if not valid_url and hasattr(ie, '_make_valid_url'): - valid_url = ie._make_valid_url() - if valid_url: - s += f' _VALID_URL = {valid_url!r}\n' return s + '\n'.join(extra_ie_code(ie, attr_base)) -- cgit v1.2.3 From 115add43876964956917bf596c1d0b148c5b3c26 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 9 Aug 2022 01:08:47 +0530 Subject: [devscripts] Create `utils` and refactor --- devscripts/make_issue_template.py | 40 ++++++++++++++++++------------------- devscripts/make_lazy_extractors.py | 16 +++++---------- devscripts/make_readme.py | 23 ++++++++++++--------- devscripts/make_supportedsites.py | 12 ++--------- devscripts/prepare_manpage.py | 41 +++++++++++++++++++------------------- devscripts/update-formulae.py | 14 +++++++------ devscripts/update-version.py | 41 +++++++++++++++++++------------------- devscripts/utils.py | 35 ++++++++++++++++++++++++++++++++ 8 files changed, 125 insertions(+), 97 deletions(-) create mode 100644 devscripts/utils.py (limited to 'devscripts') diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 90e7e0b43..fd964c6c6 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -7,20 +7,14 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import optparse import re - -def read(fname): - with open(fname, encoding='utf-8') as f: - return f.read() - - -# Get the version without importing the package -def read_version(fname): - exec(compile(read(fname), fname, 'exec')) - return locals()['__version__'] - +from devscripts.utils import ( + get_filename_args, + read_file, + read_version, + write_file, +) VERBOSE_TMPL = ''' - type: checkboxes @@ -58,20 +52,24 @@ VERBOSE_TMPL = ''' required: true '''.strip() +NO_SKIP = ''' + - type: checkboxes + attributes: + label: DO NOT REMOVE OR SKIP THE ISSUE TEMPLATE + description: Fill all fields even if you think it is irrelevant for the issue + options: + - label: I understand that I will be **blocked** if I remove or skip any mandatory\\* field + required: true +'''.strip() -def main(): - parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') - _, args = parser.parse_args() - if len(args) != 2: - parser.error('Expected an input and an output filename') - fields = {'version': read_version('yt_dlp/version.py')} +def main(): + fields = {'version': read_version(), 'no_skip': NO_SKIP} fields['verbose'] = VERBOSE_TMPL % fields fields['verbose_optional'] = re.sub(r'(\n\s+validations:)?\n\s+required: true', '', fields['verbose']) - infile, outfile = args - with open(outfile, 'w', encoding='utf-8') as outf: - outf.write(read(infile) % fields) + infile, outfile = get_filename_args(has_infile=True) + write_file(outfile, read_file(infile) % fields) if __name__ == '__main__': diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index c9fdfb562..01bd88ae6 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -7,9 +7,10 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import optparse from inspect import getsource +from devscripts.utils import get_filename_args, read_file, write_file + NO_ATTR = object() STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_NETRC_MACHINE', 'age_limit'] CLASS_METHODS = [ @@ -19,17 +20,11 @@ IE_TEMPLATE = ''' class {name}({bases}): _module = {module!r} ''' -with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: - MODULE_TEMPLATE = f.read() +MODULE_TEMPLATE = read_file('devscripts/lazy_load_template.py') def main(): - parser = optparse.OptionParser(usage='%prog [OUTFILE.py]') - args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py'] - if len(args) != 1: - parser.error('Expected only an output filename') - - lazy_extractors_filename = args[0] + lazy_extractors_filename = get_filename_args(default_outfile='yt_dlp/extractor/lazy_extractors.py') if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) @@ -46,8 +41,7 @@ def main(): *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), )) - with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: - f.write(f'{module_src}\n') + write_file(lazy_extractors_filename, f'{module_src}\n') def get_all_ies(): diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index f2e08d7c6..767ea5409 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -5,10 +5,17 @@ yt-dlp --help | make_readme.py This must be run in a console of correct width """ +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import functools import re -import sys + +from devscripts.utils import read_file, write_file README_FILE = 'README.md' @@ -63,12 +70,10 @@ PATCHES = ( ), ) -with open(README_FILE, encoding='utf-8') as f: - readme = f.read() +readme = read_file(README_FILE) -with open(README_FILE, 'w', encoding='utf-8') as f: - f.write(''.join(( - take_section(readme, end=f'## {OPTIONS_START}'), - functools.reduce(apply_patch, PATCHES, options), - take_section(readme, f'# {OPTIONS_END}'), - ))) +write_file(README_FILE, ''.join(( + take_section(readme, end=f'## {OPTIONS_START}'), + functools.reduce(apply_patch, PATCHES, options), + take_section(readme, f'# {OPTIONS_END}'), +))) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index e46f7af56..01548ef97 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -7,21 +7,13 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import optparse - +from devscripts.utils import get_filename_args, write_file from yt_dlp.extractor import list_extractor_classes def main(): - parser = optparse.OptionParser(usage='%prog OUTFILE.md') - _, args = parser.parse_args() - if len(args) != 1: - parser.error('Expected an output filename') - out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False) - - with open(args[0], 'w', encoding='utf-8') as outf: - outf.write(f'# Supported sites\n{out}\n') + write_file(get_filename_args(), f'# Supported sites\n{out}\n') if __name__ == '__main__': diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index cea934949..9b12e71e5 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,9 +1,22 @@ #!/usr/bin/env python3 -import optparse +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + import os.path import re +from devscripts.utils import ( + compose_functions, + get_filename_args, + read_file, + write_file, +) + ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') @@ -22,25 +35,6 @@ yt\-dlp \- A youtube-dl fork with additional features and patches ''' -def main(): - parser = optparse.OptionParser(usage='%prog OUTFILE.md') - _, args = parser.parse_args() - if len(args) != 1: - parser.error('Expected an output filename') - - outfile, = args - - with open(README_FILE, encoding='utf-8') as f: - readme = f.read() - - readme = filter_excluded_sections(readme) - readme = move_sections(readme) - readme = filter_options(readme) - - with open(outfile, 'w', encoding='utf-8') as outf: - outf.write(PREFIX + readme) - - def filter_excluded_sections(readme): EXCLUDED_SECTION_BEGIN_STRING = re.escape('') EXCLUDED_SECTION_END_STRING = re.escape('') @@ -92,5 +86,12 @@ def filter_options(readme): return readme.replace(section, options, 1) +TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options) + + +def main(): + write_file(get_filename_args(), PREFIX + TRANSFORM(read_file(README_FILE))) + + if __name__ == '__main__': main() diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py index 96b56b932..e79297f53 100644 --- a/devscripts/update-formulae.py +++ b/devscripts/update-formulae.py @@ -1,5 +1,10 @@ #!/usr/bin/env python3 +""" +Usage: python3 ./devscripts/update-formulae.py +version can be either 0-aligned (yt-dlp version) or normalized (PyPi version) +""" + # Allow direct execution import os import sys @@ -11,8 +16,7 @@ import json import re import urllib.request -# usage: python3 ./devscripts/update-formulae.py -# version can be either 0-aligned (yt-dlp version) or normalized (PyPl version) +from devscripts.utils import read_file, write_file filename, version = sys.argv[1:] @@ -27,11 +31,9 @@ tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('. sha256sum = tarball_file['digests']['sha256'] url = tarball_file['url'] -with open(filename) as r: - formulae_text = r.read() +formulae_text = read_file(filename) formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text, count=1) formulae_text = re.sub(r'url "[^"]*?"', 'url "%s"' % url, formulae_text, count=1) -with open(filename, 'w') as w: - w.write(formulae_text) +write_file(filename, formulae_text) diff --git a/devscripts/update-version.py b/devscripts/update-version.py index c5bc83de9..c55dd371c 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -7,32 +7,35 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import contextlib import subprocess import sys from datetime import datetime -with open('yt_dlp/version.py') as f: - exec(compile(f.read(), 'yt_dlp/version.py', 'exec')) -old_version = locals()['__version__'] +from devscripts.utils import read_version, write_file -old_version_list = old_version.split('.') -old_ver = '.'.join(old_version_list[:3]) -old_rev = old_version_list[3] if len(old_version_list) > 3 else '' +def get_new_version(revision): + version = datetime.utcnow().strftime('%Y.%m.%d') -ver = datetime.utcnow().strftime("%Y.%m.%d") + if revision: + assert revision.isdigit(), 'Revision must be a number' + else: + old_version = read_version().split('.') + if version.split('.') == old_version[:3]: + revision = str(int((old_version + [0])[3]) + 1) -rev = (sys.argv[1:] or [''])[0] # Use first argument, if present as revision number -if not rev: - rev = str(int(old_rev or 0) + 1) if old_ver == ver else '' + return f'{version}.{revision}' if revision else version -VERSION = '.'.join((ver, rev)) if rev else ver -try: - sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE) - GIT_HEAD = sp.communicate()[0].decode().strip() or None -except Exception: - GIT_HEAD = None +def get_git_head(): + with contextlib.suppress(Exception): + sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE) + return sp.communicate()[0].decode().strip() or None + + +VERSION = get_new_version((sys.argv + [''])[1]) +GIT_HEAD = get_git_head() VERSION_FILE = f'''\ # Autogenerated by devscripts/update-version.py @@ -42,8 +45,6 @@ __version__ = {VERSION!r} RELEASE_GIT_HEAD = {GIT_HEAD!r} ''' -with open('yt_dlp/version.py', 'wt') as f: - f.write(VERSION_FILE) - -print('::set-output name=ytdlp_version::' + VERSION) +write_file('yt_dlp/version.py', VERSION_FILE) +print(f'::set-output name=ytdlp_version::{VERSION}') print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}') diff --git a/devscripts/utils.py b/devscripts/utils.py new file mode 100644 index 000000000..aa17a5f7f --- /dev/null +++ b/devscripts/utils.py @@ -0,0 +1,35 @@ +import argparse +import functools + + +def read_file(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +def write_file(fname, content): + with open(fname, 'w', encoding='utf-8') as f: + return f.write(content) + + +# Get the version without importing the package +def read_version(fname='yt_dlp/version.py'): + exec(compile(read_file(fname), fname, 'exec')) + return locals()['__version__'] + + +def get_filename_args(has_infile=False, default_outfile=None): + parser = argparse.ArgumentParser() + if has_infile: + parser.add_argument('infile', help='Input file') + kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} + parser.add_argument('outfile', **kwargs, help='Output file') + + opts = parser.parse_args() + if has_infile: + return opts.infile, opts.outfile + return opts.outfile + + +def compose_functions(*functions): + return lambda x: functools.reduce(lambda y, f: f(y), functions, x) -- cgit v1.2.3 From 70b2340909d8d917f71d20181614fd7392d3f7f0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 29 Jul 2022 20:33:01 +0530 Subject: [build, devscripts] Add devscript to set a build variant Closes #4471 --- devscripts/make_readme.py | 4 ++++ devscripts/set-variant.py | 36 ++++++++++++++++++++++++++++++++++++ devscripts/update-version.py | 4 ++++ 3 files changed, 44 insertions(+) create mode 100644 devscripts/set-variant.py (limited to 'devscripts') diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 767ea5409..fad993a19 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -45,6 +45,10 @@ switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group()) delim = f'\n{" " * switch_col_width}' PATCHES = ( + ( # Standardize update message + r'(?m)^( -U, --update\s+).+(\n \s.+)*$', + r'\1Update this program to the latest version', + ), ( # Headings r'(?m)^ (\w.+\n)( (?=\w))?', r'## \1' diff --git a/devscripts/set-variant.py b/devscripts/set-variant.py new file mode 100644 index 000000000..10341e744 --- /dev/null +++ b/devscripts/set-variant.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +import argparse +import functools +import re + +from devscripts.utils import compose_functions, read_file, write_file + +VERSION_FILE = 'yt_dlp/version.py' + + +def parse_options(): + parser = argparse.ArgumentParser(description='Set the build variant of the package') + parser.add_argument('variant', help='Name of the variant') + parser.add_argument('-M', '--update-message', default=None, help='Message to show in -U') + return parser.parse_args() + + +def property_setter(name, value): + return functools.partial(re.sub, rf'(?m)^{name}\s*=\s*.+$', f'{name} = {value!r}') + + +opts = parse_options() +transform = compose_functions( + property_setter('VARIANT', opts.variant), + property_setter('UPDATE_HINT', opts.update_message) +) + +write_file(VERSION_FILE, transform(read_file(VERSION_FILE))) diff --git a/devscripts/update-version.py b/devscripts/update-version.py index c55dd371c..caebf4241 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -43,6 +43,10 @@ VERSION_FILE = f'''\ __version__ = {VERSION!r} RELEASE_GIT_HEAD = {GIT_HEAD!r} + +VARIANT = None + +UPDATE_HINT = None ''' write_file('yt_dlp/version.py', VERSION_FILE) -- cgit v1.2.3 From 96623ab5c6cea59c22395a47f00a13d334de6106 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 11 Aug 2022 07:12:20 +0530 Subject: [devscripts] Fix import Closes #4603 --- devscripts/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 devscripts/__init__.py (limited to 'devscripts') diff --git a/devscripts/__init__.py b/devscripts/__init__.py new file mode 100644 index 000000000..750dbdca7 --- /dev/null +++ b/devscripts/__init__.py @@ -0,0 +1 @@ +# Empty file needed to make devscripts.utils properly importable from outside -- cgit v1.2.3 From e5458d1d88fcc81011ab19ba610c4b37946c9fa9 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 24 Aug 2022 15:10:21 +0530 Subject: Fix lazy extractor bug in fe7866d0ed6bfa3904ce12b049a3424fdc0ea1fa and add test Fixes https://github.com/yt-dlp/yt-dlp/pull/3234#issuecomment-1225347071 --- devscripts/lazy_load_template.py | 11 +++++++---- devscripts/make_lazy_extractors.py | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index a6e26b6f6..626b85d62 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -11,14 +11,17 @@ from ..utils import ( # These bloat the lazy_extractors, so allow them to passthrough silently ALLOWED_CLASSMETHODS = {'get_testcases', 'extract_from_webpage'} +_WARNED = False class LazyLoadMetaClass(type): def __getattr__(cls, name): - if '_real_class' not in cls.__dict__ and name not in ALLOWED_CLASSMETHODS: - write_string( - 'WARNING: Falling back to normal extractor since lazy extractor ' - f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') + global _WARNED + if ('_real_class' not in cls.__dict__ + and name not in ALLOWED_CLASSMETHODS and not _WARNED): + _WARNED = True + write_string('WARNING: Falling back to normal extractor since lazy extractor ' + f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') return getattr(cls.real_class, name) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 01bd88ae6..43885331f 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -12,7 +12,9 @@ from inspect import getsource from devscripts.utils import get_filename_args, read_file, write_file NO_ATTR = object() -STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_NETRC_MACHINE', 'age_limit'] +STATIC_CLASS_PROPERTIES = [ + 'IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_ENABLED', '_NETRC_MACHINE', 'age_limit' +] CLASS_METHODS = [ 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable' ] -- cgit v1.2.3 From d2c8aadf799a63aaa7da81ae03052b1ec2addd20 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 1 Sep 2022 16:49:03 +0530 Subject: [cleanup] Misc Closes #4710, Closes #4754, Closes #4723 Authored by: pukkandan, MrRawes, DavidH-2022 --- devscripts/run_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'devscripts') diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index d496a092b..faa642e96 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -if [ -z $1 ]; then +if [ -z "$1" ]; then test_set='test' -elif [ $1 = 'core' ]; then +elif [ "$1" = 'core' ]; then test_set="-m not download" -elif [ $1 = 'download' ]; then +elif [ "$1" = 'download' ]; then test_set="-m download" else - echo 'Invalid test type "'$1'". Use "core" | "download"' + echo 'Invalid test type "'"$1"'". Use "core" | "download"' exit 1 fi -- cgit v1.2.3 From 46d72cd2c7fced093189babb484d53766f52ef57 Mon Sep 17 00:00:00 2001 From: josanabr Date: Sun, 18 Sep 2022 09:32:28 -0500 Subject: [devscripts] make_lazy_extractors: Fix for Docker (#4958) Authored by: josanabr --- devscripts/make_lazy_extractors.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'devscripts') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 43885331f..383c7e057 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -3,6 +3,7 @@ # Allow direct execution import os import sys +import shutil sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -50,12 +51,13 @@ def get_all_ies(): PLUGINS_DIRNAME = 'ytdlp_plugins' BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked' if os.path.exists(PLUGINS_DIRNAME): - os.rename(PLUGINS_DIRNAME, BLOCKED_DIRNAME) + # os.rename cannot be used, e.g. in Docker. See https://github.com/yt-dlp/yt-dlp/pull/4958 + shutil.move(PLUGINS_DIRNAME, BLOCKED_DIRNAME) try: from yt_dlp.extractor.extractors import _ALL_CLASSES finally: if os.path.exists(BLOCKED_DIRNAME): - os.rename(BLOCKED_DIRNAME, PLUGINS_DIRNAME) + shutil.move(BLOCKED_DIRNAME, PLUGINS_DIRNAME) return _ALL_CLASSES -- cgit v1.2.3 From 2fa669f759eae6d5c7e608e3ee628f9d60d03e83 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Thu, 22 Sep 2022 01:37:44 +0530 Subject: [docs] Misc improvements Closes #4987, Closes #4906, Closes #4919, Closes #4977, Closes #4979 --- devscripts/make_lazy_extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devscripts') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 383c7e057..2d4530eb9 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -2,8 +2,8 @@ # Allow direct execution import os -import sys import shutil +import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -- cgit v1.2.3 From 7d61d2306e36d31ad992df4e332be4ff8c708ef8 Mon Sep 17 00:00:00 2001 From: Lesmiscore Date: Fri, 21 Oct 2022 22:26:00 +0900 Subject: [build] Replace `set-output` with `GITHUB_OUTPUT` (#5315) https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ Authored by: Lesmiscore --- devscripts/update-version.py | 4 +++- devscripts/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'devscripts') diff --git a/devscripts/update-version.py b/devscripts/update-version.py index caebf4241..9cf8b42e6 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -50,5 +50,7 @@ UPDATE_HINT = None ''' write_file('yt_dlp/version.py', VERSION_FILE) -print(f'::set-output name=ytdlp_version::{VERSION}') +github_output = os.getenv('GITHUB_OUTPUT') +if github_output: + write_file(github_output, f'ytdlp_version={VERSION}\n', 'a') print(f'\nVersion = {VERSION}, Git HEAD = {GIT_HEAD}') diff --git a/devscripts/utils.py b/devscripts/utils.py index aa17a5f7f..b91b8e65a 100644 --- a/devscripts/utils.py +++ b/devscripts/utils.py @@ -7,8 +7,8 @@ def read_file(fname): return f.read() -def write_file(fname, content): - with open(fname, 'w', encoding='utf-8') as f: +def write_file(fname, content, mode='w'): + with open(fname, mode, encoding='utf-8') as f: return f.write(content) -- cgit v1.2.3 From 6368e2e639bca7e66609911d2672b6a9dc65b052 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 16 Nov 2022 06:27:43 +0530 Subject: [cleanup] Misc Closes #5541 --- devscripts/lazy_load_template.py | 2 +- devscripts/make_lazy_extractors.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index 626b85d62..c8815e01b 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -10,7 +10,7 @@ from ..utils import ( ) # These bloat the lazy_extractors, so allow them to passthrough silently -ALLOWED_CLASSMETHODS = {'get_testcases', 'extract_from_webpage'} +ALLOWED_CLASSMETHODS = {'extract_from_webpage', 'get_testcases', 'get_webpage_testcases'} _WARNED = False diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 2d4530eb9..c502bdf89 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -14,10 +14,17 @@ from devscripts.utils import get_filename_args, read_file, write_file NO_ATTR = object() STATIC_CLASS_PROPERTIES = [ - 'IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_VALID_URL', '_WORKING', '_ENABLED', '_NETRC_MACHINE', 'age_limit' + 'IE_NAME', '_ENABLED', '_VALID_URL', # Used for URL matching + '_WORKING', 'IE_DESC', '_NETRC_MACHINE', 'SEARCH_KEY', # Used for --extractor-descriptions + 'age_limit', # Used for --age-limit (evaluated) + '_RETURN_TYPE', # Accessed in CLI only with instance (evaluated) ] CLASS_METHODS = [ - 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable' + 'ie_key', 'suitable', '_match_valid_url', # Used for URL matching + 'working', 'get_temp_id', '_match_id', # Accessed just before instance creation + 'description', # Used for --extractor-descriptions + 'is_suitable', # Used for --age-limit + 'supports_login', 'is_single_video', # Accessed in CLI only with instance ] IE_TEMPLATE = ''' class {name}({bases}): -- cgit v1.2.3