diff options
Diffstat (limited to 'devscripts')
| -rw-r--r-- | devscripts/__init__.py | 1 | ||||
| -rwxr-xr-x | devscripts/bash-completion.py | 9 | ||||
| -rw-r--r-- | devscripts/buildserver.py | 435 | ||||
| -rw-r--r-- | devscripts/check-porn.py | 21 | ||||
| -rwxr-xr-x | devscripts/fish-completion.py | 12 | ||||
| -rw-r--r-- | devscripts/generate_aes_testdata.py | 12 | ||||
| -rw-r--r-- | devscripts/lazy_load_template.py | 39 | ||||
| -rwxr-xr-x | devscripts/make_contributing.py | 6 | ||||
| -rw-r--r-- | devscripts/make_lazy_extractors.py | 213 | ||||
| -rw-r--r--[-rwxr-xr-x] | devscripts/make_readme.py | 90 | ||||
| -rw-r--r-- | devscripts/make_supportedsites.py | 42 | ||||
| -rwxr-xr-x | devscripts/posix-locale.sh | 6 | ||||
| -rw-r--r-- | devscripts/prepare_manpage.py | 43 | ||||
| -rw-r--r-- | devscripts/run_tests.bat | 1 | ||||
| -rwxr-xr-x | devscripts/run_tests.sh | 12 | ||||
| -rw-r--r-- | devscripts/set-variant.py | 36 | ||||
| -rw-r--r-- | devscripts/utils.py | 35 | ||||
| -rwxr-xr-x | devscripts/zsh-completion.py | 9 |
18 files changed, 356 insertions, 666 deletions
diff --git a/devscripts/__init__.py b/devscripts/__init__.py new file mode 100644 index 0000000..750dbdc --- /dev/null +++ b/devscripts/__init__.py @@ -0,0 +1 @@ +# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index e0768d2..cef5414 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals +# Allow direct execution import os -from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + import hypervideo_dl BASH_COMPLETION_FILE = "completions/bash/hypervideo" @@ -26,5 +27,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = hypervideo_dl.parseOpts()[0] +parser = hypervideo_dl.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py deleted file mode 100644 index 2a8039e..0000000 --- a/devscripts/buildserver.py +++ /dev/null @@ -1,435 +0,0 @@ -# UNUSED - -#!/usr/bin/python3 - -import argparse -import ctypes -import functools -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback -import os.path - -sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) -from hypervideo_dl.compat import ( - compat_input, - compat_http_server, - compat_str, - compat_urlparse, -) - -# These are not used outside of buildserver.py thus not in compat.py - -try: - import winreg as compat_winreg -except ImportError: # Python 2 - import _winreg as compat_winreg - -try: - import socketserver as compat_socketserver -except ImportError: # Python 2 - import SocketServer as compat_socketserver - - -class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer): - allow_reuse_address = True - - -advapi32 = ctypes.windll.advapi32 - -SC_MANAGER_ALL_ACCESS = 0xf003f -SC_MANAGER_CREATE_SERVICE = 0x02 -SERVICE_WIN32_OWN_PROCESS = 0x10 -SERVICE_AUTO_START = 0x2 -SERVICE_ERROR_NORMAL = 0x1 -DELETE = 0x00010000 -SERVICE_STATUS_START_PENDING = 0x00000002 -SERVICE_STATUS_RUNNING = 0x00000004 -SERVICE_ACCEPT_STOP = 0x1 - -SVCNAME = 'youtubedl_builder' - -LPTSTR = ctypes.c_wchar_p -START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR)) - - -class SERVICE_TABLE_ENTRY(ctypes.Structure): - _fields_ = [ - ('lpServiceName', LPTSTR), - ('lpServiceProc', START_CALLBACK) - ] - - -HandlerEx = ctypes.WINFUNCTYPE( - ctypes.c_int, # return - ctypes.c_int, # dwControl - ctypes.c_int, # dwEventType - ctypes.c_void_p, # lpEventData, - ctypes.c_void_p, # lpContext, -) - - -def _ctypes_array(c_type, py_array): - ar = (c_type * len(py_array))() - ar[:] = py_array - return ar - - -def win_OpenSCManager(): - res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS) - if not res: - raise Exception('Opening service manager failed - ' - 'are you running this as administrator?') - return res - - -def win_install_service(service_name, cmdline): - manager = win_OpenSCManager() - try: - h = advapi32.CreateServiceW( - manager, service_name, None, - SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, - cmdline, None, None, None, None, None) - if not h: - raise OSError('Service creation failed: %s' % ctypes.FormatError()) - - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_uninstall_service(service_name): - manager = win_OpenSCManager() - try: - h = advapi32.OpenServiceW(manager, service_name, DELETE) - if not h: - raise OSError('Could not find service %s: %s' % ( - service_name, ctypes.FormatError())) - - try: - if not advapi32.DeleteService(h): - raise OSError('Deletion failed: %s' % ctypes.FormatError()) - finally: - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_service_report_event(service_name, msg, is_error=True): - with open('C:/sshkeys/log', 'a', encoding='utf-8') as f: - f.write(msg + '\n') - - event_log = advapi32.RegisterEventSourceW(None, service_name) - if not event_log: - raise OSError('Could not report event: %s' % ctypes.FormatError()) - - try: - type_id = 0x0001 if is_error else 0x0004 - event_id = 0xc0000000 if is_error else 0x40000000 - lines = _ctypes_array(LPTSTR, [msg]) - - if not advapi32.ReportEventW( - event_log, type_id, 0, event_id, None, len(lines), 0, - lines, None): - raise OSError('Event reporting failed: %s' % ctypes.FormatError()) - finally: - advapi32.DeregisterEventSource(event_log) - - -def win_service_handler(stop_event, *args): - try: - raise ValueError('Handler called with args ' + repr(args)) - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_set_status(handle, status_code): - svcStatus = SERVICE_STATUS() - svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS - svcStatus.dwCurrentState = status_code - svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP - - svcStatus.dwServiceSpecificExitCode = 0 - - if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)): - raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError()) - - -def win_service_main(service_name, real_main, argc, argv_raw): - try: - # args = [argv_raw[i].value for i in range(argc)] - stop_event = threading.Event() - handler = HandlerEx(functools.partial(stop_event, win_service_handler)) - h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None) - if not h: - raise OSError('Handler registration failed: %s' % - ctypes.FormatError()) - - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_start(service_name, real_main): - try: - cb = START_CALLBACK( - functools.partial(win_service_main, service_name, real_main)) - dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [ - SERVICE_TABLE_ENTRY( - service_name, - cb - ), - SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK)) - ]) - - if not advapi32.StartServiceCtrlDispatcherW(dispatch_table): - raise OSError('ctypes start failed: %s' % ctypes.FormatError()) - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--install', - action='store_const', dest='action', const='install', - help='Launch at Windows startup') - parser.add_argument('-u', '--uninstall', - action='store_const', dest='action', const='uninstall', - help='Remove Windows service') - parser.add_argument('-s', '--service', - action='store_const', dest='action', const='service', - help='Run as a Windows service') - parser.add_argument('-b', '--bind', metavar='<host:port>', - action='store', default='0.0.0.0:8142', - help='Bind to host:port (default %default)') - options = parser.parse_args(args=args) - - if options.action == 'install': - fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox') - cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind) - win_install_service(SVCNAME, cmdline) - return - - if options.action == 'uninstall': - win_uninstall_service(SVCNAME) - return - - if options.action == 'service': - win_service_start(SVCNAME, main) - return - - host, port_str = options.bind.split(':') - port = int(port_str) - - print('Listening on %s:%d' % (host, port)) - srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler) - thr = threading.Thread(target=srv.serve_forever) - thr.start() - compat_input('Press ENTER to shut down') - srv.shutdown() - thr.join() - - -def rmtree(path): - for name in os.listdir(path): - fname = os.path.join(path, name) - if os.path.isdir(fname): - rmtree(fname) - else: - os.chmod(fname, 0o666) - os.remove(fname) - os.rmdir(path) - - -class BuildError(Exception): - def __init__(self, output, code=500): - self.output = output - self.code = code - - def __str__(self): - return self.output - - -class HTTPError(BuildError): - pass - - -class PythonBuilder(object): - def __init__(self, **kwargs): - python_version = kwargs.pop('python', '3.4') - python_path = None - for node in ('Wow6432Node\\', ''): - try: - key = compat_winreg.OpenKey( - compat_winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version)) - try: - python_path, _ = compat_winreg.QueryValueEx(key, '') - finally: - compat_winreg.CloseKey(key) - break - except Exception: - pass - - if not python_path: - raise BuildError('No such Python version: %s' % python_version) - - self.pythonPath = python_path - - super(PythonBuilder, self).__init__(**kwargs) - - -class GITInfoBuilder(object): - def __init__(self, **kwargs): - try: - self.user, self.repoName = kwargs['path'][:2] - self.rev = kwargs.pop('rev') - except ValueError: - raise BuildError('Invalid path') - except KeyError as e: - raise BuildError('Missing mandatory parameter "%s"' % e.args[0]) - - path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user) - if not os.path.exists(path): - os.makedirs(path) - self.basePath = tempfile.mkdtemp(dir=path) - self.buildPath = os.path.join(self.basePath, 'build') - - super(GITInfoBuilder, self).__init__(**kwargs) - - -class GITBuilder(GITInfoBuilder): - def build(self): - try: - subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath]) - subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(GITBuilder, self).build() - - -class YoutubeDLBuilder(object): - authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] - - def __init__(self, **kwargs): - if self.repoName != 'hypervideo': - raise BuildError('Invalid repository "%s"' % self.repoName) - if self.user not in self.authorizedUsers: - raise HTTPError('Unauthorized user "%s"' % self.user, 401) - - super(YoutubeDLBuilder, self).__init__(**kwargs) - - def build(self): - try: - proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath) - proc.wait() - #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], - # cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(YoutubeDLBuilder, self).build() - - -class DownloadBuilder(object): - def __init__(self, **kwargs): - self.handler = kwargs.pop('handler') - self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:])) - self.srcPath = os.path.abspath(os.path.normpath(self.srcPath)) - if not self.srcPath.startswith(self.buildPath): - raise HTTPError(self.srcPath, 401) - - super(DownloadBuilder, self).__init__(**kwargs) - - def build(self): - if not os.path.exists(self.srcPath): - raise HTTPError('No such file', 404) - if os.path.isdir(self.srcPath): - raise HTTPError('Is a directory: %s' % self.srcPath, 401) - - self.handler.send_response(200) - self.handler.send_header('Content-Type', 'application/octet-stream') - self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1]) - self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size)) - self.handler.end_headers() - - with open(self.srcPath, 'rb') as src: - shutil.copyfileobj(src, self.handler.wfile) - - super(DownloadBuilder, self).build() - - -class CleanupTempDir(object): - def build(self): - try: - rmtree(self.basePath) - except Exception as e: - print('WARNING deleting "%s": %s' % (self.basePath, e)) - - super(CleanupTempDir, self).build() - - -class Null(object): - def __init__(self, **kwargs): - pass - - def start(self): - pass - - def close(self): - pass - - def build(self): - pass - - -class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null): - pass - - -class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler): - actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. - - def do_GET(self): - path = compat_urlparse.urlparse(self.path) - paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()]) - action, _, path = path.path.strip('/').partition('/') - if path: - path = path.split('/') - if action in self.actionDict: - try: - builder = self.actionDict[action](path=path, handler=self, **paramDict) - builder.start() - try: - builder.build() - finally: - builder.close() - except BuildError as e: - self.send_response(e.code) - msg = compat_str(e).encode('UTF-8') - self.send_header('Content-Type', 'text/plain; charset=UTF-8') - self.send_header('Content-Length', len(msg)) - self.end_headers() - self.wfile.write(msg) - else: - self.send_response(500, 'Unknown build method "%s"' % action) - else: - self.send_response(500, 'Malformed URL') - -if __name__ == '__main__': - main() diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 7dd372f..fc72c30 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - """ This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check if we are not 'age_limit' tagging some porn site @@ -12,11 +10,14 @@ pass the list filename as the only argument # Allow direct execution import os import sys + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import urllib.parse +import urllib.request + from test.helper import gettestcases -from hypervideo_dl.utils import compat_urllib_parse_urlparse -from hypervideo_dl.utils import compat_urllib_request if len(sys.argv) > 1: METHOD = 'LIST' @@ -27,9 +28,9 @@ else: for test in gettestcases(): if METHOD == 'EURISTIC': try: - webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() + webpage = urllib.request.urlopen(test['url'], timeout=10).read() except Exception: - print('\nFail: {0}'.format(test['name'])) + print('\nFail: {}'.format(test['name'])) continue webpage = webpage.decode('utf8', 'replace') @@ -37,9 +38,9 @@ for test in gettestcases(): RESULT = 'porn' in webpage.lower() elif METHOD == 'LIST': - domain = compat_urllib_parse_urlparse(test['url']).netloc + domain = urllib.parse.urlparse(test['url']).netloc if not domain: - print('\nFail: {0}'.format(test['name'])) + print('\nFail: {}'.format(test['name'])) continue domain = '.'.join(domain.split('.')[-2:]) @@ -47,11 +48,11 @@ for test in gettestcases(): if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or test['info_dict']['age_limit'] != 18): - print('\nPotential missing age_limit check: {0}'.format(test['name'])) + print('\nPotential missing age_limit check: {}'.format(test['name'])) elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and test['info_dict']['age_limit'] == 18): - print('\nPotential false negative: {0}'.format(test['name'])) + print('\nPotential false negative: {}'.format(test['name'])) else: sys.stdout.write('.') diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 84ced2d..0b2b113 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals -import optparse +# Allow direct execution import os -from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +import optparse + import hypervideo_dl from hypervideo_dl.utils import shell_quote @@ -46,5 +48,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = hypervideo_dl.parseOpts()[0] +parser = hypervideo_dl.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index 09feeaa..f131e47 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - -import codecs -import subprocess +# Allow direct execution import os import sys + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from hypervideo_dl.utils import intlist_to_bytes + +import codecs +import subprocess + from hypervideo_dl.aes import aes_encrypt, key_expansion +from hypervideo_dl.utils import intlist_to_bytes secret_msg = b'Secret message goes here' diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index da89e07..c8815e0 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -1,31 +1,38 @@ -# coding: utf-8 +import importlib +import random import re -from ..utils import bug_reports_message, write_string +from ..utils import ( + age_restricted, + bug_reports_message, + classproperty, + write_string, +) + +# These bloat the lazy_extractors, so allow them to passthrough silently +ALLOWED_CLASSMETHODS = {'extract_from_webpage', 'get_testcases', 'get_webpage_testcases'} +_WARNED = False class LazyLoadMetaClass(type): def __getattr__(cls, name): - if '_real_class' not in cls.__dict__: - write_string( - f'WARNING: Falling back to normal extractor since lazy extractor ' - f'{cls.__name__} does not have attribute {name}{bug_reports_message()}') - return getattr(cls._get_real_class(), name) + global _WARNED + if ('_real_class' not in cls.__dict__ + and name not in ALLOWED_CLASSMETHODS and not _WARNED): + _WARNED = True + write_string('WARNING: Falling back to normal extractor since lazy extractor ' + f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') + return getattr(cls.real_class, name) class LazyLoadExtractor(metaclass=LazyLoadMetaClass): - _module = None - _WORKING = True - - @classmethod - def _get_real_class(cls): + @classproperty + def real_class(cls): if '_real_class' not in cls.__dict__: - mod = __import__(cls._module, fromlist=(cls.__name__,)) - cls._real_class = getattr(mod, cls.__name__) + cls._real_class = getattr(importlib.import_module(cls._module), cls.__name__) return cls._real_class def __new__(cls, *args, **kwargs): - real_cls = cls._get_real_class() - instance = real_cls.__new__(real_cls) + instance = cls.real_class.__new__(cls.real_class) instance.__init__(*args, **kwargs) return instance diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 8c5f107..e777730 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals -import io import optparse import re @@ -16,7 +14,7 @@ def main(): infile, outfile = args - with io.open(infile, encoding='utf-8') as inf: + with open(infile, encoding='utf-8') as inf: readme = inf.read() bug_text = re.search( @@ -26,7 +24,7 @@ def main(): out = bug_text + dev_text - with io.open(outfile, 'w', encoding='utf-8') as outf: + with open(outfile, 'w', encoding='utf-8') as outf: outf.write(out) diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 1e22620..69e1758 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,105 +1,128 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals, print_function -from inspect import getsource -import io +# Allow direct execution import os -from os.path import dirname as dirn +import shutil import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) - -lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'hypervideo_dl/extractor/lazy_extractors.py' -if os.path.exists(lazy_extractors_filename): - os.remove(lazy_extractors_filename) - -# Block plugins from loading -plugins_dirname = 'ytdlp_plugins' -plugins_blocked_dirname = 'ytdlp_plugins_blocked' -if os.path.exists(plugins_dirname): - os.rename(plugins_dirname, plugins_blocked_dirname) - -from hypervideo_dl.extractor import _ALL_CLASSES -from hypervideo_dl.extractor.common import InfoExtractor, SearchInfoExtractor +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -if os.path.exists(plugins_blocked_dirname): - os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py', 'rt') as f: - module_template = f.read() - -CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] -module_contents = [ - module_template, - *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES], - '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] +from inspect import getsource -ie_template = ''' +from devscripts.utils import get_filename_args, read_file, write_file + +NO_ATTR = object() +STATIC_CLASS_PROPERTIES = [ + 'IE_NAME', '_ENABLED', '_VALID_URL', # Used for URL matching + '_WORKING', 'IE_DESC', '_NETRC_MACHINE', 'SEARCH_KEY', # Used for --extractor-descriptions + 'age_limit', # Used for --age-limit (evaluated) + '_RETURN_TYPE', # Accessed in CLI only with instance (evaluated) +] +CLASS_METHODS = [ + 'ie_key', 'suitable', '_match_valid_url', # Used for URL matching + 'working', 'get_temp_id', '_match_id', # Accessed just before instance creation + 'description', # Used for --extractor-descriptions + 'is_suitable', # Used for --age-limit + 'supports_login', 'is_single_video', # Accessed in CLI only with instance +] +IE_TEMPLATE = ''' class {name}({bases}): - _module = '{module}' + _module = {module!r} ''' - - -def get_base_name(base): - if base is InfoExtractor: - return 'LazyLoadExtractor' - elif base is SearchInfoExtractor: - return 'LazyLoadSearchExtractor' - else: - return base.__name__ - - -def build_lazy_ie(ie, name): - s = ie_template.format( - name=name, - bases=', '.join(map(get_base_name, ie.__bases__)), - module=ie.__module__) - valid_url = getattr(ie, '_VALID_URL', None) - if not valid_url and hasattr(ie, '_make_valid_url'): - valid_url = ie._make_valid_url() - if valid_url: - s += f' _VALID_URL = {valid_url!r}\n' - if not ie._WORKING: - s += ' _WORKING = False\n' - if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += f'\n{getsource(ie.suitable)}' - return s - - -# find the correct sorting and add the required base classes so that subclasses -# can be correctly created -classes = _ALL_CLASSES[:-1] -ordered_cls = [] -while classes: - for c in classes[:]: - bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor)) - stop = False - for b in bases: - if b not in classes and b not in ordered_cls: - if b.__name__ == 'GenericIE': - exit() - classes.insert(0, b) - stop = True - if stop: - break - if all(b in ordered_cls for b in bases): - ordered_cls.append(c) - classes.remove(c) - break -ordered_cls.append(_ALL_CLASSES[-1]) - -names = [] -for ie in ordered_cls: - name = ie.__name__ - src = build_lazy_ie(ie, name) - module_contents.append(src) - if ie in _ALL_CLASSES: - names.append(name) - -module_contents.append( - '\n_ALL_CLASSES = [{0}]'.format(', '.join(names))) - -module_src = '\n'.join(module_contents) + '\n' - -with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: - f.write(module_src) +MODULE_TEMPLATE = read_file('devscripts/lazy_load_template.py') + + +def main(): + lazy_extractors_filename = get_filename_args(default_outfile='hypervideo_dl/extractor/lazy_extractors.py') + if os.path.exists(lazy_extractors_filename): + os.remove(lazy_extractors_filename) + + _ALL_CLASSES = get_all_ies() # Must be before import + + from hypervideo_dl.extractor.common import InfoExtractor, SearchInfoExtractor + + DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR}) + module_src = '\n'.join(( + MODULE_TEMPLATE, + ' _module = None', + *extra_ie_code(DummyInfoExtractor), + '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', + *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), + )) + + write_file(lazy_extractors_filename, f'{module_src}\n') + + +def get_all_ies(): + PLUGINS_DIRNAME = 'ytdlp_plugins' + BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked' + if os.path.exists(PLUGINS_DIRNAME): + # os.rename cannot be used, e.g. in Docker. See https://github.com/hypervideo/hypervideo/pull/4958 + shutil.move(PLUGINS_DIRNAME, BLOCKED_DIRNAME) + try: + from hypervideo_dl.extractor.extractors import _ALL_CLASSES + finally: + if os.path.exists(BLOCKED_DIRNAME): + shutil.move(BLOCKED_DIRNAME, PLUGINS_DIRNAME) + return _ALL_CLASSES + + +def extra_ie_code(ie, base=None): + for var in STATIC_CLASS_PROPERTIES: + val = getattr(ie, var) + if val != (getattr(base, var) if base else NO_ATTR): + yield f' {var} = {val!r}' + yield '' + + for name in CLASS_METHODS: + f = getattr(ie, name) + if not base or f.__func__ != getattr(base, name).__func__: + yield getsource(f) + + +def build_ies(ies, bases, attr_base): + names = [] + for ie in sort_ies(ies, bases): + yield build_lazy_ie(ie, ie.__name__, attr_base) + if ie in ies: + names.append(ie.__name__) + + yield f'\n_ALL_CLASSES = [{", ".join(names)}]' + + +def sort_ies(ies, ignored_bases): + """find the correct sorting and add the required base classes so that subclasses can be correctly created""" + classes, returned_classes = ies[:-1], set() + assert ies[-1].__name__ == 'GenericIE', 'Last IE must be GenericIE' + while classes: + for c in classes[:]: + bases = set(c.__bases__) - {object, *ignored_bases} + restart = False + for b in sorted(bases, key=lambda x: x.__name__): + if b not in classes and b not in returned_classes: + assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE' + classes.insert(0, b) + restart = True + if restart: + break + if bases <= returned_classes: + yield c + returned_classes.add(c) + classes.remove(c) + break + yield ies[-1] + + +def build_lazy_ie(ie, name, attr_base): + bases = ', '.join({ + 'InfoExtractor': 'LazyLoadExtractor', + 'SearchInfoExtractor': 'LazyLoadSearchExtractor', + }.get(base.__name__, base.__name__) for base in ie.__bases__) + + s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases) + return s + '\n'.join(extra_ie_code(ie, attr_base)) + + +if __name__ == '__main__': + main() diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 1a9a017..6adfca0 100755..100644 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,31 +1,83 @@ #!/usr/bin/env python3 -# hypervideo --help | make_readme.py -# This must be run in a console of correct width +""" +hypervideo --help | make_readme.py +This must be run in a console of correct width +""" -from __future__ import unicode_literals - -import io +# Allow direct execution +import os import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +import functools import re +from devscripts.utils import read_file, write_file + README_FILE = 'README.md' -helptext = sys.stdin.read() -if isinstance(helptext, bytes): - helptext = helptext.decode('utf-8') +OPTIONS_START = 'General Options:' +OPTIONS_END = 'CONFIGURATION' +EPILOG_START = 'See full documentation' +ALLOWED_OVERSHOOT = 2 + +DISABLE_PATCH = object() + + +def take_section(text, start=None, end=None, *, shift=0): + return text[ + text.index(start) + shift if start else None: + text.index(end) + shift if end else None + ] + + +def apply_patch(text, patch): + return text if patch[0] is DISABLE_PATCH else re.sub(*patch, text) + + +options = take_section(sys.stdin.read(), f'\n {OPTIONS_START}', f'\n{EPILOG_START}', shift=1) -with io.open(README_FILE, encoding='utf-8') as f: - oldreadme = f.read() +max_width = max(map(len, options.split('\n'))) +switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group()) +delim = f'\n{" " * switch_col_width}' -header = oldreadme[:oldreadme.index('# OPTIONS')] -footer = oldreadme[oldreadme.index('# CONFIGURATION'):] +PATCHES = ( + ( # Standardize update message + r'(?m)^( -U, --update\s+).+(\n \s.+)*$', + r'\1Update this program to the latest version', + ), + ( # Headings + r'(?m)^ (\w.+\n)( (?=\w))?', + r'## \1' + ), + ( # Do not split URLs + rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s', + lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n')) + ), + ( # Do not split "words" + rf'(?m)({delim}\S+)+$', + lambda mobj: ''.join((delim, mobj.group(0).replace(delim, ''))) + ), + ( # Allow overshooting last line + rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})', + lambda mobj: (mobj.group().replace(delim, ' ') + if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT + else mobj.group()) + ), + ( # Avoid newline when a space is available b/w switch and description + DISABLE_PATCH, # This creates issues with prepare_manpage + r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim), + r'\1 ' + ), +) -options = helptext[helptext.index(' General Options:') + 19:] -options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) -options = '# OPTIONS\n' + options + '\n' +readme = read_file(README_FILE) -with io.open(README_FILE, 'w', encoding='utf-8') as f: - f.write(header) - f.write(options) - f.write(footer) +write_file(README_FILE, ''.join(( + take_section(readme, end=f'## {OPTIONS_START}'), + functools.reduce(apply_patch, PATCHES, options), + take_section(readme, f'# {OPTIONS_END}'), +))) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 9bce04b..5ccc75d 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,47 +1,19 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals -import io -import optparse +# Allow direct execution import os import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# Import hypervideo_dl -ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') -sys.path.insert(0, ROOT_DIR) -import hypervideo_dl + +from devscripts.utils import get_filename_args, write_file +from hypervideo_dl.extractor import list_extractor_classes def main(): - parser = optparse.OptionParser(usage='%prog OUTFILE.md') - options, args = parser.parse_args() - if len(args) != 1: - parser.error('Expected an output filename') - - outfile, = args - - def gen_ies_md(ies): - for ie in ies: - ie_md = '**{0}**'.format(ie.IE_NAME) - if ie.IE_DESC is False: - continue - if ie.IE_DESC is not None: - ie_md += ': {0}'.format(ie.IE_DESC) - search_key = getattr(ie, 'SEARCH_KEY', None) - if search_key is not None: - ie_md += f'; "{ie.SEARCH_KEY}:" prefix' - if not ie.working(): - ie_md += ' (Currently broken)' - yield ie_md - - ies = sorted(hypervideo_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()) - out = '# Supported sites\n' + ''.join( - ' - ' + md + '\n' - for md in gen_ies_md(ies)) - - with io.open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False) + write_file(get_filename_args(), f'# Supported sites\n{out}\n') if __name__ == '__main__': diff --git a/devscripts/posix-locale.sh b/devscripts/posix-locale.sh deleted file mode 100755 index 0aa7a59..0000000 --- a/devscripts/posix-locale.sh +++ /dev/null @@ -1,6 +0,0 @@ - -# source this file in your shell to get a POSIX locale (which will break many programs, but that's kind of the point) - -export LC_ALL=POSIX -export LANG=POSIX -export LANGUAGE=POSIX diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 8920df1..ef41d21 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,11 +1,22 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals -import io -import optparse +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + import os.path import re +from devscripts.utils import ( + compose_functions, + get_filename_args, + read_file, + write_file, +) + ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') @@ -24,25 +35,6 @@ yt\-dlp \- A youtube-dl fork with additional features and patches ''' -def main(): - parser = optparse.OptionParser(usage='%prog OUTFILE.md') - options, args = parser.parse_args() - if len(args) != 1: - parser.error('Expected an output filename') - - outfile, = args - - with io.open(README_FILE, encoding='utf-8') as f: - readme = f.read() - - readme = filter_excluded_sections(readme) - readme = move_sections(readme) - readme = filter_options(readme) - - with io.open(outfile, 'w', encoding='utf-8') as outf: - outf.write(PREFIX + readme) - - def filter_excluded_sections(readme): EXCLUDED_SECTION_BEGIN_STRING = re.escape('<!-- MANPAGE: BEGIN EXCLUDED SECTION -->') EXCLUDED_SECTION_END_STRING = re.escape('<!-- MANPAGE: END EXCLUDED SECTION -->') @@ -94,5 +86,12 @@ def filter_options(readme): return readme.replace(section, options, 1) +TRANSFORM = compose_functions(filter_excluded_sections, move_sections, filter_options) + + +def main(): + write_file(get_filename_args(), PREFIX + TRANSFORM(read_file(README_FILE))) + + if __name__ == '__main__': main() diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat index b8bb393..190d239 100644 --- a/devscripts/run_tests.bat +++ b/devscripts/run_tests.bat @@ -13,4 +13,5 @@ if ["%~1"]==[""] ( exit /b 1 ) +set PYTHONWARNINGS=error pytest %test_set% diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index c9a75ba..faa642e 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -1,14 +1,14 @@ -#!/bin/sh +#!/usr/bin/env sh -if [ -z $1 ]; then +if [ -z "$1" ]; then test_set='test' -elif [ $1 = 'core' ]; then +elif [ "$1" = 'core' ]; then test_set="-m not download" -elif [ $1 = 'download' ]; then +elif [ "$1" = 'download' ]; then test_set="-m download" else - echo 'Invalid test type "'$1'". Use "core" | "download"' + echo 'Invalid test type "'"$1"'". Use "core" | "download"' exit 1 fi -python3 -m pytest "$test_set" +python3 -bb -Werror -m pytest "$test_set" diff --git a/devscripts/set-variant.py b/devscripts/set-variant.py new file mode 100644 index 0000000..c9c8561 --- /dev/null +++ b/devscripts/set-variant.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +# Allow direct execution +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +import argparse +import functools +import re + +from devscripts.utils import compose_functions, read_file, write_file + +VERSION_FILE = 'hypervideo_dl/version.py' + + +def parse_options(): + parser = argparse.ArgumentParser(description='Set the build variant of the package') + parser.add_argument('variant', help='Name of the variant') + parser.add_argument('-M', '--update-message', default=None, help='Message to show in -U') + return parser.parse_args() + + +def property_setter(name, value): + return functools.partial(re.sub, rf'(?m)^{name}\s*=\s*.+$', f'{name} = {value!r}') + + +opts = parse_options() +transform = compose_functions( + property_setter('VARIANT', opts.variant), + property_setter('UPDATE_HINT', opts.update_message) +) + +write_file(VERSION_FILE, transform(read_file(VERSION_FILE))) diff --git a/devscripts/utils.py b/devscripts/utils.py new file mode 100644 index 0000000..3f67e62 --- /dev/null +++ b/devscripts/utils.py @@ -0,0 +1,35 @@ +import argparse +import functools + + +def read_file(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +def write_file(fname, content, mode='w'): + with open(fname, mode, encoding='utf-8') as f: + return f.write(content) + + +# Get the version without importing the package +def read_version(fname='hypervideo_dl/version.py'): + exec(compile(read_file(fname), fname, 'exec')) + return locals()['__version__'] + + +def get_filename_args(has_infile=False, default_outfile=None): + parser = argparse.ArgumentParser() + if has_infile: + parser.add_argument('infile', help='Input file') + kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} + parser.add_argument('outfile', **kwargs, help='Output file') + + opts = parser.parse_args() + if has_infile: + return opts.infile, opts.outfile + return opts.outfile + + +def compose_functions(*functions): + return lambda x: functools.reduce(lambda y, f: f(y), functions, x) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index c8620a5..4012ae0 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals +# Allow direct execution import os -from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + import hypervideo_dl ZSH_COMPLETION_FILE = "completions/zsh/_hypervideo" @@ -45,5 +46,5 @@ def build_completion(opt_parser): f.write(template) -parser = hypervideo_dl.parseOpts()[0] +parser = hypervideo_dl.parseOpts(ignore_config_files=True)[0] build_completion(parser) |