From 86e5f3ed2e6e71eb81ea4c9e26288f16119ffd0c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Apr 2022 20:40:28 +0530 Subject: [cleanup] Upgrade syntax Using https://github.com/asottile/pyupgrade 1. `__future__` imports and `coding: utf-8` were removed 2. Files were rewritten with `pyupgrade --py36-plus --keep-percent-format` 3. f-strings were cherry-picked from `pyupgrade --py36-plus` Extractors are left untouched (except removing header) to avoid unnecessary merge conflicts --- devscripts/make_lazy_extractors.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index b58fb85e3..24e8cfa5b 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals, print_function - from inspect import getsource -import io import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' if os.path.exists(lazy_extractors_filename): @@ -25,7 +22,7 @@ from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor if os.path.exists(plugins_blocked_dirname): os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py', 'rt') as f: +with open('devscripts/lazy_load_template.py') as f: module_template = f.read() CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] @@ -72,7 +69,7 @@ classes = _ALL_CLASSES[:-1] ordered_cls = [] while classes: for c in classes[:]: - bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor)) + bases = set(c.__bases__) - {object, InfoExtractor, SearchInfoExtractor} stop = False for b in bases: if b not in classes and b not in ordered_cls: @@ -97,9 +94,9 @@ for ie in ordered_cls: names.append(name) module_contents.append( - '\n_ALL_CLASSES = [{0}]'.format(', '.join(names))) + '\n_ALL_CLASSES = [{}]'.format(', '.join(names))) module_src = '\n'.join(module_contents) + '\n' -with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: +with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: f.write(module_src) -- cgit v1.2.3 From f82711587cee043cb2496fe180b5cc0e07c06eda Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 04:02:57 +0530 Subject: [cleanup] Sort imports Using https://github.com/PyCQA/isort isort -m VERTICAL_HANGING_INDENT --py 36 -l 80 --rr -n --tc . --- devscripts/make_lazy_extractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 24e8cfa5b..6d5f96cf0 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -from inspect import getsource import os -from os.path import dirname as dirn import sys +from inspect import getsource +from os.path import dirname as dirn sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) -- cgit v1.2.3 From e5a998f3684e7c56f9cf1c07c4e176e891d96509 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 05:31:54 +0530 Subject: [cleanup] Misc cleanup (#2173) Authored by: fstirlitz, pukkandan --- devscripts/make_lazy_extractors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 6d5f96cf0..5e2070602 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -2,9 +2,8 @@ import os import sys from inspect import getsource -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' if os.path.exists(lazy_extractors_filename): -- cgit v1.2.3 From c1714454313e01c94a7e55e1cb99d439ff933a43 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 27 Apr 2022 13:45:45 +0530 Subject: [cleanup,build] Cleanup some build-related code Fixes an issue in 7ab56be2c7309a2d11d4ee28c71f8fb29da21ef7 --- devscripts/make_lazy_extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5e2070602..6dc8fed90 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -21,7 +21,7 @@ from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor if os.path.exists(plugins_blocked_dirname): os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py') as f: +with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: module_template = f.read() CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] -- cgit v1.2.3 From 82d020804de938bb7e87bd6bbc4961757b892cd0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 11 May 2022 21:24:44 +0530 Subject: [extractor] Use classmethod/property where possible and refactor lazy extractors accordingly. This reduces the need to create extractor instances --- devscripts/make_lazy_extractors.py | 196 +++++++++++++++++++++---------------- 1 file changed, 110 insertions(+), 86 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 6dc8fed90..8ddc54b9b 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,101 +1,125 @@ #!/usr/bin/env python3 import os +import optparse import sys from inspect import getsource sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' -if os.path.exists(lazy_extractors_filename): - os.remove(lazy_extractors_filename) -# Block plugins from loading -plugins_dirname = 'ytdlp_plugins' -plugins_blocked_dirname = 'ytdlp_plugins_blocked' -if os.path.exists(plugins_dirname): - os.rename(plugins_dirname, plugins_blocked_dirname) - -from yt_dlp.extractor import _ALL_CLASSES -from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor - -if os.path.exists(plugins_blocked_dirname): - os.rename(plugins_blocked_dirname, plugins_dirname) - -with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: - module_template = f.read() - -CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] -module_contents = [ - module_template, - *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES], - '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] - -ie_template = ''' +NO_ATTR = object() +STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE'] +CLASS_METHODS = [ + 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', +] +IE_TEMPLATE = ''' class {name}({bases}): - _module = '{module}' + _module = {module!r} ''' - - -def get_base_name(base): - if base is InfoExtractor: - return 'LazyLoadExtractor' - elif base is SearchInfoExtractor: - return 'LazyLoadSearchExtractor' - else: - return base.__name__ - - -def build_lazy_ie(ie, name): - s = ie_template.format( - name=name, - bases=', '.join(map(get_base_name, ie.__bases__)), - module=ie.__module__) +with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: + MODULE_TEMPLATE = f.read() + + +def main(): + parser = optparse.OptionParser(usage='%prog [OUTFILE.py]') + args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py'] + if len(args) != 1: + parser.error('Expected only an output filename') + + lazy_extractors_filename = args[0] + if os.path.exists(lazy_extractors_filename): + os.remove(lazy_extractors_filename) + + _ALL_CLASSES = get_all_ies() # Must be before import + + from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor + + DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR}) + module_src = '\n'.join(( + MODULE_TEMPLATE, + ' _module = None', + *extra_ie_code(DummyInfoExtractor), + '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', + *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), + )) + + with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: + f.write(f'{module_src}\n') + + +def get_all_ies(): + PLUGINS_DIRNAME = 'ytdlp_plugins' + BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked' + if os.path.exists(PLUGINS_DIRNAME): + os.rename(PLUGINS_DIRNAME, BLOCKED_DIRNAME) + try: + from yt_dlp.extractor import _ALL_CLASSES + finally: + if os.path.exists(BLOCKED_DIRNAME): + os.rename(BLOCKED_DIRNAME, PLUGINS_DIRNAME) + return _ALL_CLASSES + + +def extra_ie_code(ie, base=None): + for var in STATIC_CLASS_PROPERTIES: + val = getattr(ie, var) + if val != (getattr(base, var) if base else NO_ATTR): + yield f' {var} = {val!r}' + yield '' + + for name in CLASS_METHODS: + f = getattr(ie, name) + if not base or f.__func__ != getattr(base, name).__func__: + yield getsource(f) + + +def build_ies(ies, bases, attr_base): + names = [] + for ie in sort_ies(ies, bases): + yield build_lazy_ie(ie, ie.__name__, attr_base) + if ie in ies: + names.append(ie.__name__) + + yield f'\n_ALL_CLASSES = [{", ".join(names)}]' + + +def sort_ies(ies, ignored_bases): + """find the correct sorting and add the required base classes so that subclasses can be correctly created""" + classes, returned_classes = ies[:-1], set() + assert ies[-1].__name__ == 'GenericIE', 'Last IE must be GenericIE' + while classes: + for c in classes[:]: + bases = set(c.__bases__) - {object, *ignored_bases} + restart = False + for b in bases: + if b not in classes and b not in returned_classes: + assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE' + classes.insert(0, b) + restart = True + if restart: + break + if bases <= returned_classes: + yield c + returned_classes.add(c) + classes.remove(c) + break + yield ies[-1] + + +def build_lazy_ie(ie, name, attr_base): + bases = ', '.join({ + 'InfoExtractor': 'LazyLoadExtractor', + 'SearchInfoExtractor': 'LazyLoadSearchExtractor', + }.get(base.__name__, base.__name__) for base in ie.__bases__) + + s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases) valid_url = getattr(ie, '_VALID_URL', None) if not valid_url and hasattr(ie, '_make_valid_url'): valid_url = ie._make_valid_url() if valid_url: s += f' _VALID_URL = {valid_url!r}\n' - if not ie._WORKING: - s += ' _WORKING = False\n' - if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += f'\n{getsource(ie.suitable)}' - return s - - -# find the correct sorting and add the required base classes so that subclasses -# can be correctly created -classes = _ALL_CLASSES[:-1] -ordered_cls = [] -while classes: - for c in classes[:]: - bases = set(c.__bases__) - {object, InfoExtractor, SearchInfoExtractor} - stop = False - for b in bases: - if b not in classes and b not in ordered_cls: - if b.__name__ == 'GenericIE': - exit() - classes.insert(0, b) - stop = True - if stop: - break - if all(b in ordered_cls for b in bases): - ordered_cls.append(c) - classes.remove(c) - break -ordered_cls.append(_ALL_CLASSES[-1]) - -names = [] -for ie in ordered_cls: - name = ie.__name__ - src = build_lazy_ie(ie, name) - module_contents.append(src) - if ie in _ALL_CLASSES: - names.append(name) - -module_contents.append( - '\n_ALL_CLASSES = [{}]'.format(', '.join(names))) - -module_src = '\n'.join(module_contents) + '\n' - -with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: - f.write(module_src) + return s + '\n'.join(extra_ie_code(ie, attr_base)) + + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 241464919271278831f23b3a086dcf57aeb80d3b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 16 May 2022 19:36:36 +0530 Subject: [cleanup] Misc cleanup --- devscripts/make_lazy_extractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 8ddc54b9b..8c481bc2d 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -8,9 +8,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) NO_ATTR = object() -STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE'] +STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit'] CLASS_METHODS = [ - 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', + 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable' ] IE_TEMPLATE = ''' class {name}({bases}): -- cgit v1.2.3