From ab96d1ad1bcdb943aa6519980e5383ca91f7da2b Mon Sep 17 00:00:00 2001 From: felix Date: Sun, 19 Sep 2021 13:16:11 +0200 Subject: [cleanup] Remove unused scripts/tests (#2173) Authored by fstirlitz, pukkandan --- devscripts/buildserver.py | 435 ------------------------ devscripts/create-github-release.py | 112 ------ devscripts/gh-pages.unused/add-version.py | 43 --- devscripts/gh-pages.unused/generate-download.py | 22 -- devscripts/gh-pages.unused/sign-versions.py | 34 -- devscripts/gh-pages.unused/update-copyright.py | 21 -- devscripts/gh-pages.unused/update-feed.py | 76 ----- devscripts/gh-pages.unused/update-sites.py | 37 -- devscripts/posix-locale.sh | 6 - devscripts/release.sh | 143 -------- devscripts/show-downloads-statistics.py | 49 --- devscripts/wine-py2exe.sh | 58 ---- 12 files changed, 1036 deletions(-) delete mode 100644 devscripts/buildserver.py delete mode 100644 devscripts/create-github-release.py delete mode 100644 devscripts/gh-pages.unused/add-version.py delete mode 100644 devscripts/gh-pages.unused/generate-download.py delete mode 100644 devscripts/gh-pages.unused/sign-versions.py delete mode 100644 devscripts/gh-pages.unused/update-copyright.py delete mode 100644 devscripts/gh-pages.unused/update-feed.py delete mode 100644 devscripts/gh-pages.unused/update-sites.py delete mode 100755 devscripts/posix-locale.sh delete mode 100755 devscripts/release.sh delete mode 100644 devscripts/show-downloads-statistics.py delete mode 100755 devscripts/wine-py2exe.sh (limited to 'devscripts') diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py deleted file mode 100644 index cd544b816..000000000 --- a/devscripts/buildserver.py +++ /dev/null @@ -1,435 +0,0 @@ -# UNUSED - -#!/usr/bin/python3 - -import argparse -import ctypes -import functools -import shutil -import subprocess -import sys -import tempfile -import threading -import traceback -import os.path - -sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) -from yt_dlp.compat import ( - compat_input, - compat_http_server, - compat_str, - compat_urlparse, -) - -# These are not used outside of buildserver.py thus not in compat.py - -try: - import winreg as compat_winreg -except ImportError: # Python 2 - import _winreg as compat_winreg - -try: - import socketserver as compat_socketserver -except ImportError: # Python 2 - import SocketServer as compat_socketserver - - -class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer): - allow_reuse_address = True - - -advapi32 = ctypes.windll.advapi32 - -SC_MANAGER_ALL_ACCESS = 0xf003f -SC_MANAGER_CREATE_SERVICE = 0x02 -SERVICE_WIN32_OWN_PROCESS = 0x10 -SERVICE_AUTO_START = 0x2 -SERVICE_ERROR_NORMAL = 0x1 -DELETE = 0x00010000 -SERVICE_STATUS_START_PENDING = 0x00000002 -SERVICE_STATUS_RUNNING = 0x00000004 -SERVICE_ACCEPT_STOP = 0x1 - -SVCNAME = 'youtubedl_builder' - -LPTSTR = ctypes.c_wchar_p -START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR)) - - -class SERVICE_TABLE_ENTRY(ctypes.Structure): - _fields_ = [ - ('lpServiceName', LPTSTR), - ('lpServiceProc', START_CALLBACK) - ] - - -HandlerEx = ctypes.WINFUNCTYPE( - ctypes.c_int, # return - ctypes.c_int, # dwControl - ctypes.c_int, # dwEventType - ctypes.c_void_p, # lpEventData, - ctypes.c_void_p, # lpContext, -) - - -def _ctypes_array(c_type, py_array): - ar = (c_type * len(py_array))() - ar[:] = py_array - return ar - - -def win_OpenSCManager(): - res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS) - if not res: - raise Exception('Opening service manager failed - ' - 'are you running this as administrator?') - return res - - -def win_install_service(service_name, cmdline): - manager = win_OpenSCManager() - try: - h = advapi32.CreateServiceW( - manager, service_name, None, - SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, - cmdline, None, None, None, None, None) - if not h: - raise OSError('Service creation failed: %s' % ctypes.FormatError()) - - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_uninstall_service(service_name): - manager = win_OpenSCManager() - try: - h = advapi32.OpenServiceW(manager, service_name, DELETE) - if not h: - raise OSError('Could not find service %s: %s' % ( - service_name, ctypes.FormatError())) - - try: - if not advapi32.DeleteService(h): - raise OSError('Deletion failed: %s' % ctypes.FormatError()) - finally: - advapi32.CloseServiceHandle(h) - finally: - advapi32.CloseServiceHandle(manager) - - -def win_service_report_event(service_name, msg, is_error=True): - with open('C:/sshkeys/log', 'a', encoding='utf-8') as f: - f.write(msg + '\n') - - event_log = advapi32.RegisterEventSourceW(None, service_name) - if not event_log: - raise OSError('Could not report event: %s' % ctypes.FormatError()) - - try: - type_id = 0x0001 if is_error else 0x0004 - event_id = 0xc0000000 if is_error else 0x40000000 - lines = _ctypes_array(LPTSTR, [msg]) - - if not advapi32.ReportEventW( - event_log, type_id, 0, event_id, None, len(lines), 0, - lines, None): - raise OSError('Event reporting failed: %s' % ctypes.FormatError()) - finally: - advapi32.DeregisterEventSource(event_log) - - -def win_service_handler(stop_event, *args): - try: - raise ValueError('Handler called with args ' + repr(args)) - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_set_status(handle, status_code): - svcStatus = SERVICE_STATUS() - svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS - svcStatus.dwCurrentState = status_code - svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP - - svcStatus.dwServiceSpecificExitCode = 0 - - if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)): - raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError()) - - -def win_service_main(service_name, real_main, argc, argv_raw): - try: - # args = [argv_raw[i].value for i in range(argc)] - stop_event = threading.Event() - handler = HandlerEx(functools.partial(stop_event, win_service_handler)) - h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None) - if not h: - raise OSError('Handler registration failed: %s' % - ctypes.FormatError()) - - TODO - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def win_service_start(service_name, real_main): - try: - cb = START_CALLBACK( - functools.partial(win_service_main, service_name, real_main)) - dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [ - SERVICE_TABLE_ENTRY( - service_name, - cb - ), - SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK)) - ]) - - if not advapi32.StartServiceCtrlDispatcherW(dispatch_table): - raise OSError('ctypes start failed: %s' % ctypes.FormatError()) - except Exception as e: - tb = traceback.format_exc() - msg = str(e) + '\n' + tb - win_service_report_event(service_name, msg, is_error=True) - raise - - -def main(args=None): - parser = argparse.ArgumentParser() - parser.add_argument('-i', '--install', - action='store_const', dest='action', const='install', - help='Launch at Windows startup') - parser.add_argument('-u', '--uninstall', - action='store_const', dest='action', const='uninstall', - help='Remove Windows service') - parser.add_argument('-s', '--service', - action='store_const', dest='action', const='service', - help='Run as a Windows service') - parser.add_argument('-b', '--bind', metavar='', - action='store', default='0.0.0.0:8142', - help='Bind to host:port (default %default)') - options = parser.parse_args(args=args) - - if options.action == 'install': - fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox') - cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind) - win_install_service(SVCNAME, cmdline) - return - - if options.action == 'uninstall': - win_uninstall_service(SVCNAME) - return - - if options.action == 'service': - win_service_start(SVCNAME, main) - return - - host, port_str = options.bind.split(':') - port = int(port_str) - - print('Listening on %s:%d' % (host, port)) - srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler) - thr = threading.Thread(target=srv.serve_forever) - thr.start() - compat_input('Press ENTER to shut down') - srv.shutdown() - thr.join() - - -def rmtree(path): - for name in os.listdir(path): - fname = os.path.join(path, name) - if os.path.isdir(fname): - rmtree(fname) - else: - os.chmod(fname, 0o666) - os.remove(fname) - os.rmdir(path) - - -class BuildError(Exception): - def __init__(self, output, code=500): - self.output = output - self.code = code - - def __str__(self): - return self.output - - -class HTTPError(BuildError): - pass - - -class PythonBuilder(object): - def __init__(self, **kwargs): - python_version = kwargs.pop('python', '3.4') - python_path = None - for node in ('Wow6432Node\\', ''): - try: - key = compat_winreg.OpenKey( - compat_winreg.HKEY_LOCAL_MACHINE, - r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version)) - try: - python_path, _ = compat_winreg.QueryValueEx(key, '') - finally: - compat_winreg.CloseKey(key) - break - except Exception: - pass - - if not python_path: - raise BuildError('No such Python version: %s' % python_version) - - self.pythonPath = python_path - - super(PythonBuilder, self).__init__(**kwargs) - - -class GITInfoBuilder(object): - def __init__(self, **kwargs): - try: - self.user, self.repoName = kwargs['path'][:2] - self.rev = kwargs.pop('rev') - except ValueError: - raise BuildError('Invalid path') - except KeyError as e: - raise BuildError('Missing mandatory parameter "%s"' % e.args[0]) - - path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user) - if not os.path.exists(path): - os.makedirs(path) - self.basePath = tempfile.mkdtemp(dir=path) - self.buildPath = os.path.join(self.basePath, 'build') - - super(GITInfoBuilder, self).__init__(**kwargs) - - -class GITBuilder(GITInfoBuilder): - def build(self): - try: - subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath]) - subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(GITBuilder, self).build() - - -class YoutubeDLBuilder(object): - authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] - - def __init__(self, **kwargs): - if self.repoName != 'yt-dlp': - raise BuildError('Invalid repository "%s"' % self.repoName) - if self.user not in self.authorizedUsers: - raise HTTPError('Unauthorized user "%s"' % self.user, 401) - - super(YoutubeDLBuilder, self).__init__(**kwargs) - - def build(self): - try: - proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath) - proc.wait() - #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], - # cwd=self.buildPath) - except subprocess.CalledProcessError as e: - raise BuildError(e.output) - - super(YoutubeDLBuilder, self).build() - - -class DownloadBuilder(object): - def __init__(self, **kwargs): - self.handler = kwargs.pop('handler') - self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:])) - self.srcPath = os.path.abspath(os.path.normpath(self.srcPath)) - if not self.srcPath.startswith(self.buildPath): - raise HTTPError(self.srcPath, 401) - - super(DownloadBuilder, self).__init__(**kwargs) - - def build(self): - if not os.path.exists(self.srcPath): - raise HTTPError('No such file', 404) - if os.path.isdir(self.srcPath): - raise HTTPError('Is a directory: %s' % self.srcPath, 401) - - self.handler.send_response(200) - self.handler.send_header('Content-Type', 'application/octet-stream') - self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1]) - self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size)) - self.handler.end_headers() - - with open(self.srcPath, 'rb') as src: - shutil.copyfileobj(src, self.handler.wfile) - - super(DownloadBuilder, self).build() - - -class CleanupTempDir(object): - def build(self): - try: - rmtree(self.basePath) - except Exception as e: - print('WARNING deleting "%s": %s' % (self.basePath, e)) - - super(CleanupTempDir, self).build() - - -class Null(object): - def __init__(self, **kwargs): - pass - - def start(self): - pass - - def close(self): - pass - - def build(self): - pass - - -class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null): - pass - - -class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler): - actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. - - def do_GET(self): - path = compat_urlparse.urlparse(self.path) - paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()]) - action, _, path = path.path.strip('/').partition('/') - if path: - path = path.split('/') - if action in self.actionDict: - try: - builder = self.actionDict[action](path=path, handler=self, **paramDict) - builder.start() - try: - builder.build() - finally: - builder.close() - except BuildError as e: - self.send_response(e.code) - msg = compat_str(e).encode('UTF-8') - self.send_header('Content-Type', 'text/plain; charset=UTF-8') - self.send_header('Content-Length', len(msg)) - self.end_headers() - self.wfile.write(msg) - else: - self.send_response(500, 'Unknown build method "%s"' % action) - else: - self.send_response(500, 'Malformed URL') - -if __name__ == '__main__': - main() diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py deleted file mode 100644 index 53b3e0f48..000000000 --- a/devscripts/create-github-release.py +++ /dev/null @@ -1,112 +0,0 @@ -# Unused - -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import io -import json -import mimetypes -import netrc -import optparse -import os -import re -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from yt_dlp.compat import ( - compat_basestring, - compat_getpass, - compat_print, - compat_urllib_request, -) -from yt_dlp.utils import ( - make_HTTPS_handler, - sanitized_Request, -) - - -class GitHubReleaser(object): - _API_URL = 'https://api.github.com/repos/ytdl-org/youtube-dl/releases' - _UPLOADS_URL = 'https://uploads.github.com/repos/ytdl-org/youtube-dl/releases/%s/assets?name=%s' - _NETRC_MACHINE = 'github.com' - - def __init__(self, debuglevel=0): - self._init_github_account() - https_handler = make_HTTPS_handler({}, debuglevel=debuglevel) - self._opener = compat_urllib_request.build_opener(https_handler) - - def _init_github_account(self): - try: - info = netrc.netrc().authenticators(self._NETRC_MACHINE) - if info is not None: - self._token = info[2] - compat_print('Using GitHub credentials found in .netrc...') - return - else: - compat_print('No GitHub credentials found in .netrc') - except (IOError, netrc.NetrcParseError): - compat_print('Unable to parse .netrc') - self._token = compat_getpass( - 'Type your GitHub PAT (personal access token) and press [Return]: ') - - def _call(self, req): - if isinstance(req, compat_basestring): - req = sanitized_Request(req) - req.add_header('Authorization', 'token %s' % self._token) - response = self._opener.open(req).read().decode('utf-8') - return json.loads(response) - - def list_releases(self): - return self._call(self._API_URL) - - def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False): - data = { - 'tag_name': tag_name, - 'target_commitish': 'master', - 'name': name, - 'body': body, - 'draft': draft, - 'prerelease': prerelease, - } - req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8')) - return self._call(req) - - def create_asset(self, release_id, asset): - asset_name = os.path.basename(asset) - url = self._UPLOADS_URL % (release_id, asset_name) - # Our files are small enough to be loaded directly into memory. - data = open(asset, 'rb').read() - req = sanitized_Request(url, data) - mime_type, _ = mimetypes.guess_type(asset_name) - req.add_header('Content-Type', mime_type or 'application/octet-stream') - return self._call(req) - - -def main(): - parser = optparse.OptionParser(usage='%prog CHANGELOG VERSION BUILDPATH') - options, args = parser.parse_args() - if len(args) != 3: - parser.error('Expected a version and a build directory') - - changelog_file, version, build_path = args - - with io.open(changelog_file, encoding='utf-8') as inf: - changelog = inf.read() - - mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog) - body = mobj.group(1) if mobj else '' - - releaser = GitHubReleaser() - - new_release = releaser.create_release( - version, name='yt-dlp %s' % version, body=body) - release_id = new_release['id'] - - for asset in os.listdir(build_path): - compat_print('Uploading %s...' % asset) - releaser.create_asset(release_id, os.path.join(build_path, asset)) - - -if __name__ == '__main__': - main() diff --git a/devscripts/gh-pages.unused/add-version.py b/devscripts/gh-pages.unused/add-version.py deleted file mode 100644 index 9ea01374d..000000000 --- a/devscripts/gh-pages.unused/add-version.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import json -import sys -import hashlib -import os.path - - -if len(sys.argv) <= 1: - print('Specify the version number as parameter') - sys.exit() -version = sys.argv[1] - -with open('update/LATEST_VERSION', 'w') as f: - f.write(version) - -versions_info = json.load(open('update/versions.json')) -if 'signature' in versions_info: - del versions_info['signature'] - -new_version = {} - -filenames = { - 'bin': 'yt-dlp', - 'exe': 'yt-dlp.exe', - 'tar': 'yt-dlp-%s.tar.gz' % version} -build_dir = os.path.join('..', '..', 'build', version) -for key, filename in filenames.items(): - url = 'https://yt-dl.org/downloads/%s/%s' % (version, filename) - fn = os.path.join(build_dir, filename) - with open(fn, 'rb') as f: - data = f.read() - if not data: - raise ValueError('File %s is empty!' % fn) - sha256sum = hashlib.sha256(data).hexdigest() - new_version[key] = (url, sha256sum) - -versions_info['versions'][version] = new_version -versions_info['latest'] = version - -with open('update/versions.json', 'w') as jsonf: - json.dump(versions_info, jsonf, indent=4, sort_keys=True) diff --git a/devscripts/gh-pages.unused/generate-download.py b/devscripts/gh-pages.unused/generate-download.py deleted file mode 100644 index a873d32ee..000000000 --- a/devscripts/gh-pages.unused/generate-download.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import json - -versions_info = json.load(open('update/versions.json')) -version = versions_info['latest'] -version_dict = versions_info['versions'][version] - -# Read template page -with open('download.html.in', 'r', encoding='utf-8') as tmplf: - template = tmplf.read() - -template = template.replace('@PROGRAM_VERSION@', version) -template = template.replace('@PROGRAM_URL@', version_dict['bin'][0]) -template = template.replace('@PROGRAM_SHA256SUM@', version_dict['bin'][1]) -template = template.replace('@EXE_URL@', version_dict['exe'][0]) -template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1]) -template = template.replace('@TAR_URL@', version_dict['tar'][0]) -template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1]) -with open('download.html', 'w', encoding='utf-8') as dlf: - dlf.write(template) diff --git a/devscripts/gh-pages.unused/sign-versions.py b/devscripts/gh-pages.unused/sign-versions.py deleted file mode 100644 index fa389c358..000000000 --- a/devscripts/gh-pages.unused/sign-versions.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import unicode_literals, with_statement - -import rsa -import json -from binascii import hexlify - -try: - input = raw_input -except NameError: - pass - -versions_info = json.load(open('update/versions.json')) -if 'signature' in versions_info: - del versions_info['signature'] - -print('Enter the PKCS1 private key, followed by a blank line:') -privkey = b'' -while True: - try: - line = input() - except EOFError: - break - if line == '': - break - privkey += line.encode('ascii') + b'\n' -privkey = rsa.PrivateKey.load_pkcs1(privkey) - -signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode() -print('signature: ' + signature) - -versions_info['signature'] = signature -with open('update/versions.json', 'w') as versionsf: - json.dump(versions_info, versionsf, indent=4, sort_keys=True) diff --git a/devscripts/gh-pages.unused/update-copyright.py b/devscripts/gh-pages.unused/update-copyright.py deleted file mode 100644 index e122d0283..000000000 --- a/devscripts/gh-pages.unused/update-copyright.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import with_statement, unicode_literals - -import datetime -import glob -import io # For Python 2 compatibility -import os -import re - -year = str(datetime.datetime.now().year) -for fn in glob.glob('*.html*'): - with io.open(fn, encoding='utf-8') as f: - content = f.read() - newc = re.sub(r'(?PCopyright © 2011-)(?P[0-9]{4})', 'Copyright © 2011-' + year, content) - if content != newc: - tmpFn = fn + '.part' - with io.open(tmpFn, 'wt', encoding='utf-8') as outf: - outf.write(newc) - os.rename(tmpFn, fn) diff --git a/devscripts/gh-pages.unused/update-feed.py b/devscripts/gh-pages.unused/update-feed.py deleted file mode 100644 index c9f2fdb07..000000000 --- a/devscripts/gh-pages.unused/update-feed.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import datetime -import io -import json -import textwrap - - -atom_template = textwrap.dedent("""\ - - - - yt-dlp releases - https://yt-dl.org/feed/yt-dlp-updates-feed - @TIMESTAMP@ - @ENTRIES@ - """) - -entry_template = textwrap.dedent(""" - - https://yt-dl.org/feed/yt-dlp-updates-feed/yt-dlp-@VERSION@ - New version @VERSION@ - - -
- Downloads available at https://yt-dl.org/downloads/@VERSION@/ -
- - - The yt-dlp maintainers - - @TIMESTAMP@ - - """) - -now = datetime.datetime.now() -now_iso = now.isoformat() + 'Z' - -atom_template = atom_template.replace('@TIMESTAMP@', now_iso) - -versions_info = json.load(open('update/versions.json')) -versions = list(versions_info['versions'].keys()) -versions.sort() - -entries = [] -for v in versions: - fields = v.split('.') - year, month, day = map(int, fields[:3]) - faked = 0 - patchlevel = 0 - while True: - try: - datetime.date(year, month, day) - except ValueError: - day -= 1 - faked += 1 - assert day > 0 - continue - break - if len(fields) >= 4: - try: - patchlevel = int(fields[3]) - except ValueError: - patchlevel = 1 - timestamp = '%04d-%02d-%02dT00:%02d:%02dZ' % (year, month, day, faked, patchlevel) - - entry = entry_template.replace('@TIMESTAMP@', timestamp) - entry = entry.replace('@VERSION@', v) - entries.append(entry) - -entries_str = textwrap.indent(''.join(entries), '\t') -atom_template = atom_template.replace('@ENTRIES@', entries_str) - -with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: - atom_file.write(atom_template) diff --git a/devscripts/gh-pages.unused/update-sites.py b/devscripts/gh-pages.unused/update-sites.py deleted file mode 100644 index b53685fcc..000000000 --- a/devscripts/gh-pages.unused/update-sites.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import sys -import os -import textwrap - -# We must be able to import yt_dlp -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - -import yt_dlp - - -def main(): - with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: - template = tmplf.read() - - ie_htmls = [] - for ie in yt_dlp.list_extractors(age_limit=None): - ie_html = '{}'.format(ie.IE_NAME) - ie_desc = getattr(ie, 'IE_DESC', None) - if ie_desc is False: - continue - elif ie_desc is not None: - ie_html += ': {}'.format(ie.IE_DESC) - if not ie.working(): - ie_html += ' (Currently broken)' - ie_htmls.append('
  • {}
    • '.format(ie_html)) - - template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t')) - - with open('supportedsites.html', 'w', encoding='utf-8') as sitesf: - sitesf.write(template) - - -if __name__ == '__main__': - main() diff --git a/devscripts/posix-locale.sh b/devscripts/posix-locale.sh deleted file mode 100755 index 0aa7a592d..000000000 --- a/devscripts/posix-locale.sh +++ /dev/null @@ -1,6 +0,0 @@ - -# source this file in your shell to get a POSIX locale (which will break many programs, but that's kind of the point) - -export LC_ALL=POSIX -export LANG=POSIX -export LANGUAGE=POSIX diff --git a/devscripts/release.sh b/devscripts/release.sh deleted file mode 100755 index 188b166e6..000000000 --- a/devscripts/release.sh +++ /dev/null @@ -1,143 +0,0 @@ -# Unused - -#!/bin/bash - -# IMPORTANT: the following assumptions are made -# * the GH repo is on the origin remote -# * the gh-pages branch is named so locally -# * the git config user.signingkey is properly set - -# You will need -# pip install coverage nose rsa wheel - -# TODO -# release notes -# make hash on local files - -set -e - -skip_tests=true -gpg_sign_commits="" -buildserver='localhost:8142' - -while true -do -case "$1" in - --run-tests) - skip_tests=false - shift - ;; - --gpg-sign-commits|-S) - gpg_sign_commits="-S" - shift - ;; - --buildserver) - buildserver="$2" - shift 2 - ;; - --*) - echo "ERROR: unknown option $1" - exit 1 - ;; - *) - break - ;; -esac -done - -if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi -version="$1" -major_version=$(echo "$version" | sed -n 's#^\([0-9]*\.[0-9]*\.[0-9]*\).*#\1#p') -if test "$major_version" '!=' "$(date '+%Y.%m.%d')"; then - echo "$version does not start with today's date!" - exit 1 -fi - -if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi -if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi -useless_files=$(find yt_dlp -type f -not -name '*.py') -if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in yt_dlp: $useless_files"; exit 1; fi -if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi -if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi -if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi -if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi - -read -p "Is Changelog up to date? (y/n) " -n 1 -if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi - -/bin/echo -e "\n### First of all, testing..." -make clean -if $skip_tests ; then - echo 'SKIPPING TESTS' -else - nosetests --verbose --with-coverage --cover-package=yt_dlp --cover-html test --stop || exit 1 -fi - -/bin/echo -e "\n### Changing version in version.py..." -sed -i "s/__version__ = '.*'/__version__ = '$version'/" yt_dlp/version.py - -/bin/echo -e "\n### Changing version in Changelog..." -sed -i "s//$version/" Changelog.md - -/bin/echo -e "\n### Committing documentation, templates and yt_dlp/version.py..." -make README.md CONTRIBUTING.md issuetemplates supportedsites -git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md yt_dlp/version.py Changelog.md -git commit $gpg_sign_commits -m "release $version" - -/bin/echo -e "\n### Now tagging, signing and pushing..." -git tag -s -m "Release $version" "$version" -git show "$version" -read -p "Is it good, can I push? (y/n) " -n 1 -if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi -echo -MASTER=$(git rev-parse --abbrev-ref HEAD) -git push origin $MASTER:master -git push origin "$version" - -/bin/echo -e "\n### OK, now it is time to build the binaries..." -REV=$(git rev-parse HEAD) -make yt-dlp yt-dlp.tar.gz -read -p "VM running? (y/n) " -n 1 -wget "http://$buildserver/build/ytdl-org/youtube-dl/yt-dlp.exe?rev=$REV" -O yt-dlp.exe -mkdir -p "build/$version" -mv yt-dlp yt-dlp.exe "build/$version" -mv yt-dlp.tar.gz "build/$version/yt-dlp-$version.tar.gz" -RELEASE_FILES="yt-dlp yt-dlp.exe yt-dlp-$version.tar.gz" -(cd build/$version/ && md5sum $RELEASE_FILES > MD5SUMS) -(cd build/$version/ && sha1sum $RELEASE_FILES > SHA1SUMS) -(cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS) -(cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS) - -/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..." -for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done - -ROOT=$(pwd) -python devscripts/create-github-release.py Changelog.md $version "$ROOT/build/$version" - -ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" - -/bin/echo -e "\n### Now switching to gh-pages..." -git clone --branch gh-pages --single-branch . build/gh-pages -( - set -e - ORIGIN_URL=$(git config --get remote.origin.url) - cd build/gh-pages - "$ROOT/devscripts/gh-pages/add-version.py" $version - "$ROOT/devscripts/gh-pages/update-feed.py" - "$ROOT/devscripts/gh-pages/sign-versions.py" < "$ROOT/updates_key.pem" - "$ROOT/devscripts/gh-pages/generate-download.py" - "$ROOT/devscripts/gh-pages/update-copyright.py" - "$ROOT/devscripts/gh-pages/update-sites.py" - git add *.html *.html.in update - git commit $gpg_sign_commits -m "release $version" - git push "$ROOT" gh-pages - git push "$ORIGIN_URL" gh-pages -) -rm -rf build - -make pypi-files -echo "Uploading to PyPi ..." -python setup.py sdist bdist_wheel upload -make clean - -/bin/echo -e "\n### DONE!" diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py deleted file mode 100644 index 4855aa7c8..000000000 --- a/devscripts/show-downloads-statistics.py +++ /dev/null @@ -1,49 +0,0 @@ -# Unused - -#!/usr/bin/env python3 -from __future__ import unicode_literals - -import itertools -import json -import os -import re -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from yt_dlp.compat import ( - compat_print, - compat_urllib_request, -) -from yt_dlp.utils import format_bytes - - -def format_size(bytes): - return '%s (%d bytes)' % (format_bytes(bytes), bytes) - - -total_bytes = 0 - -for page in itertools.count(1): - releases = json.loads(compat_urllib_request.urlopen( - 'https://api.github.com/repos/ytdl-org/youtube-dl/releases?page=%s' % page - ).read().decode('utf-8')) - - if not releases: - break - - for release in releases: - compat_print(release['name']) - for asset in release['assets']: - asset_name = asset['name'] - total_bytes += asset['download_count'] * asset['size'] - if all(not re.match(p, asset_name) for p in ( - r'^yt-dlp$', - r'^yt-dlp-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$', - r'^yt-dlp\.exe$')): - continue - compat_print( - ' %s size: %s downloads: %d' - % (asset_name, format_size(asset['size']), asset['download_count'])) - -compat_print('total downloads traffic: %s' % format_size(total_bytes)) diff --git a/devscripts/wine-py2exe.sh b/devscripts/wine-py2exe.sh deleted file mode 100755 index 8bc8ce55b..000000000 --- a/devscripts/wine-py2exe.sh +++ /dev/null @@ -1,58 +0,0 @@ -# UNUSED - -#!/bin/bash - -# Run with as parameter a setup.py that works in the current directory -# e.g. no os.chdir() -# It will run twice, the first time will crash - -set -e - -SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" - -if [ ! -d wine-py2exe ]; then - - sudo apt-get install wine1.3 axel bsdiff - - mkdir wine-py2exe - cd wine-py2exe - export WINEPREFIX=`pwd` - - axel -a "http://www.python.org/ftp/python/2.7/python-2.7.msi" - axel -a "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" - #axel -a "http://winetricks.org/winetricks" - - # http://appdb.winehq.org/objectManager.php?sClass=version&iId=21957 - echo "Follow python setup on screen" - wine msiexec /i python-2.7.msi - - echo "Follow py2exe setup on screen" - wine py2exe-0.6.9.win32-py2.7.exe - - #echo "Follow Microsoft Visual C++ 2008 Redistributable Package setup on screen" - #bash winetricks vcrun2008 - - rm py2exe-0.6.9.win32-py2.7.exe - rm python-2.7.msi - #rm winetricks - - # http://bugs.winehq.org/show_bug.cgi?id=3591 - - mv drive_c/Python27/Lib/site-packages/py2exe/run.exe drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup - bspatch drive_c/Python27/Lib/site-packages/py2exe/run.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run.exe "$SCRIPT_DIR/SizeOfImage.patch" - mv drive_c/Python27/Lib/site-packages/py2exe/run_w.exe drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup - bspatch drive_c/Python27/Lib/site-packages/py2exe/run_w.exe.backup drive_c/Python27/Lib/site-packages/py2exe/run_w.exe "$SCRIPT_DIR/SizeOfImage_w.patch" - - cd - - -else - - export WINEPREFIX="$( cd wine-py2exe && pwd )" - -fi - -wine "C:\\Python27\\python.exe" "$1" py2exe > "py2exe.log" 2>&1 || true -echo '# Copying python27.dll' >> "py2exe.log" -cp "$WINEPREFIX/drive_c/windows/system32/python27.dll" build/bdist.win32/winexe/bundle-2.7/ -wine "C:\\Python27\\python.exe" "$1" py2exe >> "py2exe.log" 2>&1 - -- cgit v1.2.3 From 86e5f3ed2e6e71eb81ea4c9e26288f16119ffd0c Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 11 Apr 2022 20:40:28 +0530 Subject: [cleanup] Upgrade syntax Using https://github.com/asottile/pyupgrade 1. `__future__` imports and `coding: utf-8` were removed 2. Files were rewritten with `pyupgrade --py36-plus --keep-percent-format` 3. f-strings were cherry-picked from `pyupgrade --py36-plus` Extractors are left untouched (except removing header) to avoid unnecessary merge conflicts --- devscripts/bash-completion.py | 4 +--- devscripts/check-porn.py | 10 ++++------ devscripts/fish-completion.py | 4 +--- devscripts/generate_aes_testdata.py | 2 -- devscripts/lazy_load_template.py | 1 - devscripts/make_contributing.py | 7 ++----- devscripts/make_issue_template.py | 7 +++---- devscripts/make_lazy_extractors.py | 13 +++++-------- devscripts/make_readme.py | 8 ++------ devscripts/make_supportedsites.py | 9 +++------ devscripts/prepare_manpage.py | 7 ++----- devscripts/update-formulae.py | 4 +--- devscripts/update-version.py | 2 +- devscripts/zsh-completion.py | 4 +--- 14 files changed, 26 insertions(+), 56 deletions(-) (limited to 'devscripts') diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 46b4b2ff5..23a9a5781 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp BASH_COMPLETION_FILE = "completions/bash/yt-dlp" diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 50f6bebc6..6188f68ec 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - """ This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check if we are not 'age_limit' tagging some porn site @@ -29,7 +27,7 @@ for test in gettestcases(): try: webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() except Exception: - print('\nFail: {0}'.format(test['name'])) + print('\nFail: {}'.format(test['name'])) continue webpage = webpage.decode('utf8', 'replace') @@ -39,7 +37,7 @@ for test in gettestcases(): elif METHOD == 'LIST': domain = compat_urllib_parse_urlparse(test['url']).netloc if not domain: - print('\nFail: {0}'.format(test['name'])) + print('\nFail: {}'.format(test['name'])) continue domain = '.'.join(domain.split('.')[-2:]) @@ -47,11 +45,11 @@ for test in gettestcases(): if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or test['info_dict']['age_limit'] != 18): - print('\nPotential missing age_limit check: {0}'.format(test['name'])) + print('\nPotential missing age_limit check: {}'.format(test['name'])) elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and test['info_dict']['age_limit'] == 18): - print('\nPotential false negative: {0}'.format(test['name'])) + print('\nPotential false negative: {}'.format(test['name'])) else: sys.stdout.write('.') diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index fb45e0280..d958a5d6b 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import optparse import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp from yt_dlp.utils import shell_quote diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index 0979eee5b..308c74a20 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import codecs import subprocess diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index da89e070d..0058915ae 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -1,4 +1,3 @@ -# coding: utf-8 import re from ..utils import bug_reports_message, write_string diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 6b1b8219c..2562c4fd7 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,7 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - -import io import optparse import re @@ -16,7 +13,7 @@ def main(): infile, outfile = args - with io.open(infile, encoding='utf-8') as inf: + with open(infile, encoding='utf-8') as inf: readme = inf.read() bug_text = re.search( @@ -26,7 +23,7 @@ def main(): out = bug_text + dev_text - with io.open(outfile, 'w', encoding='utf-8') as outf: + with open(outfile, 'w', encoding='utf-8') as outf: outf.write(out) diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 902059231..878b94166 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import io import optparse @@ -13,7 +11,7 @@ def main(): infile, outfile = args - with io.open(infile, encoding='utf-8') as inf: + with open(infile, encoding='utf-8') as inf: issue_template_tmpl = inf.read() # Get the version from yt_dlp/version.py without importing the package @@ -22,8 +20,9 @@ def main(): out = issue_template_tmpl % {'version': locals()['__version__']} - with io.open(outfile, 'w', encoding='utf-8') as outf: + with open(outfile, 'w', encoding='utf-8') as outf: outf.write(out) + if __name__ == '__main__': main() diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index b58fb85e3..24e8cfa5b 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals, print_function - from inspect import getsource -import io import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' if os.path.exists(lazy_extractors_filename): @@ -25,7 +22,7 @@ from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor if os.path.exists(plugins_blocked_dirname): os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py', 'rt') as f: +with open('devscripts/lazy_load_template.py') as f: module_template = f.read() CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] @@ -72,7 +69,7 @@ classes = _ALL_CLASSES[:-1] ordered_cls = [] while classes: for c in classes[:]: - bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor)) + bases = set(c.__bases__) - {object, InfoExtractor, SearchInfoExtractor} stop = False for b in bases: if b not in classes and b not in ordered_cls: @@ -97,9 +94,9 @@ for ie in ordered_cls: names.append(name) module_contents.append( - '\n_ALL_CLASSES = [{0}]'.format(', '.join(names))) + '\n_ALL_CLASSES = [{}]'.format(', '.join(names))) module_src = '\n'.join(module_contents) + '\n' -with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: +with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: f.write(module_src) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 3f56af744..5d85bcc63 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -2,10 +2,6 @@ # yt-dlp --help | make_readme.py # This must be run in a console of correct width - -from __future__ import unicode_literals - -import io import sys import re @@ -15,7 +11,7 @@ helptext = sys.stdin.read() if isinstance(helptext, bytes): helptext = helptext.decode('utf-8') -with io.open(README_FILE, encoding='utf-8') as f: +with open(README_FILE, encoding='utf-8') as f: oldreadme = f.read() header = oldreadme[:oldreadme.index('## General Options:')] @@ -25,7 +21,7 @@ options = helptext[helptext.index(' General Options:'):] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) options = options + '\n' -with io.open(README_FILE, 'w', encoding='utf-8') as f: +with open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 729f60a0e..26d25704e 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,7 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - -import io import optparse import os import sys @@ -23,11 +20,11 @@ def main(): def gen_ies_md(ies): for ie in ies: - ie_md = '**{0}**'.format(ie.IE_NAME) + ie_md = f'**{ie.IE_NAME}**' if ie.IE_DESC is False: continue if ie.IE_DESC is not None: - ie_md += ': {0}'.format(ie.IE_DESC) + ie_md += f': {ie.IE_DESC}' search_key = getattr(ie, 'SEARCH_KEY', None) if search_key is not None: ie_md += f'; "{ie.SEARCH_KEY}:" prefix' @@ -40,7 +37,7 @@ def main(): ' - ' + md + '\n' for md in gen_ies_md(ies)) - with io.open(outfile, 'w', encoding='utf-8') as outf: + with open(outfile, 'w', encoding='utf-8') as outf: outf.write(out) diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 29c675f8a..91e9ebced 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,7 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - -import io import optparse import os.path import re @@ -32,14 +29,14 @@ def main(): outfile, = args - with io.open(README_FILE, encoding='utf-8') as f: + with open(README_FILE, encoding='utf-8') as f: readme = f.read() readme = filter_excluded_sections(readme) readme = move_sections(readme) readme = filter_options(readme) - with io.open(outfile, 'w', encoding='utf-8') as outf: + with open(outfile, 'w', encoding='utf-8') as outf: outf.write(PREFIX + readme) diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py index 41bc1ac7a..3a0bef52e 100644 --- a/devscripts/update-formulae.py +++ b/devscripts/update-formulae.py @@ -1,6 +1,4 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import json import os import re @@ -27,7 +25,7 @@ tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('. sha256sum = tarball_file['digests']['sha256'] url = tarball_file['url'] -with open(filename, 'r') as r: +with open(filename) as r: formulae_text = r.read() formulae_text = re.sub(r'sha256 "[0-9a-f]*?"', 'sha256 "%s"' % sha256sum, formulae_text) diff --git a/devscripts/update-version.py b/devscripts/update-version.py index 0ee7bf291..233cdaa76 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -4,7 +4,7 @@ import sys import subprocess -with open('yt_dlp/version.py', 'rt') as f: +with open('yt_dlp/version.py') as f: exec(compile(f.read(), 'yt_dlp/version.py', 'exec')) old_version = locals()['__version__'] diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 780df0de6..677fe7373 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp" -- cgit v1.2.3 From f82711587cee043cb2496fe180b5cc0e07c06eda Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 04:02:57 +0530 Subject: [cleanup] Sort imports Using https://github.com/PyCQA/isort isort -m VERTICAL_HANGING_INDENT --py 36 -l 80 --rr -n --tc . --- devscripts/bash-completion.py | 2 +- devscripts/check-porn.py | 5 +++-- devscripts/fish-completion.py | 2 +- devscripts/generate_aes_testdata.py | 6 +++--- devscripts/make_lazy_extractors.py | 4 ++-- devscripts/make_readme.py | 2 +- devscripts/make_supportedsites.py | 1 - devscripts/update-formulae.py | 1 - devscripts/update-version.py | 5 ++--- devscripts/zsh-completion.py | 2 +- 10 files changed, 14 insertions(+), 16 deletions(-) (limited to 'devscripts') diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 23a9a5781..73d698c39 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os -from os.path import dirname as dirn import sys +from os.path import dirname as dirn sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 6188f68ec..08f663e4b 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -10,11 +10,12 @@ pass the list filename as the only argument # Allow direct execution import os import sys + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import gettestcases -from yt_dlp.utils import compat_urllib_parse_urlparse -from yt_dlp.utils import compat_urllib_request + +from yt_dlp.utils import compat_urllib_parse_urlparse, compat_urllib_request if len(sys.argv) > 1: METHOD = 'LIST' diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index d958a5d6b..c318b69e4 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import optparse import os -from os.path import dirname as dirn import sys +from os.path import dirname as dirn sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index 308c74a20..c7d83f1a7 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import codecs -import subprocess - import os +import subprocess import sys + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from yt_dlp.utils import intlist_to_bytes from yt_dlp.aes import aes_encrypt, key_expansion +from yt_dlp.utils import intlist_to_bytes secret_msg = b'Secret message goes here' diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 24e8cfa5b..6d5f96cf0 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -from inspect import getsource import os -from os.path import dirname as dirn import sys +from inspect import getsource +from os.path import dirname as dirn sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 5d85bcc63..1719ac8e4 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -2,8 +2,8 @@ # yt-dlp --help | make_readme.py # This must be run in a console of correct width -import sys import re +import sys README_FILE = 'README.md' helptext = sys.stdin.read() diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 26d25704e..0a0d08f56 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -3,7 +3,6 @@ import optparse import os import sys - # Import yt_dlp ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') sys.path.insert(0, ROOT_DIR) diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py index 3a0bef52e..6424f5d9b 100644 --- a/devscripts/update-formulae.py +++ b/devscripts/update-formulae.py @@ -8,7 +8,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from yt_dlp.compat import compat_urllib_request - # usage: python3 ./devscripts/update-formulae.py # version can be either 0-aligned (yt-dlp version) or normalized (PyPl version) diff --git a/devscripts/update-version.py b/devscripts/update-version.py index 233cdaa76..991cfb2af 100644 --- a/devscripts/update-version.py +++ b/devscripts/update-version.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -from datetime import datetime -import sys import subprocess - +import sys +from datetime import datetime with open('yt_dlp/version.py') as f: exec(compile(f.read(), 'yt_dlp/version.py', 'exec')) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 677fe7373..2d5ac2a45 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import os -from os.path import dirname as dirn import sys +from os.path import dirname as dirn sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) import yt_dlp -- cgit v1.2.3 From e5a998f3684e7c56f9cf1c07c4e176e891d96509 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 12 Apr 2022 05:31:54 +0530 Subject: [cleanup] Misc cleanup (#2173) Authored by: fstirlitz, pukkandan --- devscripts/bash-completion.py | 4 ++-- devscripts/fish-completion.py | 4 ++-- devscripts/make_lazy_extractors.py | 3 +-- devscripts/zsh-completion.py | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) (limited to 'devscripts') diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 73d698c39..27ec7ca7a 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp BASH_COMPLETION_FILE = "completions/bash/yt-dlp" diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index c318b69e4..dcb1d6582 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -2,9 +2,9 @@ import optparse import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp from yt_dlp.utils import shell_quote diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 6d5f96cf0..5e2070602 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -2,9 +2,8 @@ import os import sys from inspect import getsource -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' if os.path.exists(lazy_extractors_filename): diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 2d5ac2a45..06660d8fd 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import os import sys -from os.path import dirname as dirn -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp" -- cgit v1.2.3 From b6dc37fe2aee167bf11f863f960a4888f4886718 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Apr 2022 02:12:48 +0530 Subject: [test] Convert warnings into errors * And fix some existing warnings Authored by: fstirlitz --- devscripts/run_tests.bat | 1 + devscripts/run_tests.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'devscripts') diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat index b8bb393d9..190d23918 100644 --- a/devscripts/run_tests.bat +++ b/devscripts/run_tests.bat @@ -13,4 +13,5 @@ if ["%~1"]==[""] ( exit /b 1 ) +set PYTHONWARNINGS=error pytest %test_set% diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh index c9a75ba00..e9904ae35 100755 --- a/devscripts/run_tests.sh +++ b/devscripts/run_tests.sh @@ -11,4 +11,4 @@ else exit 1 fi -python3 -m pytest "$test_set" +python3 -bb -Werror -m pytest "$test_set" -- cgit v1.2.3 From 19a0394044bfad36cd665450271b8eb048a41c02 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Apr 2022 02:28:28 +0530 Subject: [cleanup] Misc cleanup and refactor (#2173) --- devscripts/make_readme.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'devscripts') diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 1719ac8e4..1401c2e5a 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -6,22 +6,25 @@ import re import sys README_FILE = 'README.md' -helptext = sys.stdin.read() +OPTIONS_START = 'General Options:' +OPTIONS_END = 'CONFIGURATION' +EPILOG_START = 'See full documentation' + + +helptext = sys.stdin.read() if isinstance(helptext, bytes): helptext = helptext.decode('utf-8') -with open(README_FILE, encoding='utf-8') as f: - oldreadme = f.read() +start, end = helptext.index(f'\n {OPTIONS_START}'), helptext.index(f'\n{EPILOG_START}') +options = re.sub(r'(?m)^ (\w.+)$', r'## \1', helptext[start + 1: end + 1]) -header = oldreadme[:oldreadme.index('## General Options:')] -footer = oldreadme[oldreadme.index('# CONFIGURATION'):] +with open(README_FILE, encoding='utf-8') as f: + readme = f.read() -options = helptext[helptext.index(' General Options:'):] -options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) -options = options + '\n' +header = readme[:readme.index(f'## {OPTIONS_START}')] +footer = readme[readme.index(f'# {OPTIONS_END}'):] with open(README_FILE, 'w', encoding='utf-8') as f: - f.write(header) - f.write(options) - f.write(footer) + for part in (header, options, footer): + f.write(part) -- cgit v1.2.3 From c1714454313e01c94a7e55e1cb99d439ff933a43 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 27 Apr 2022 13:45:45 +0530 Subject: [cleanup,build] Cleanup some build-related code Fixes an issue in 7ab56be2c7309a2d11d4ee28c71f8fb29da21ef7 --- devscripts/bash-completion.py | 2 +- devscripts/fish-completion.py | 2 +- devscripts/make_issue_template.py | 24 +++++++++++++----------- devscripts/make_lazy_extractors.py | 2 +- devscripts/zsh-completion.py | 2 +- 5 files changed, 17 insertions(+), 15 deletions(-) (limited to 'devscripts') diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 27ec7ca7a..268e8a2ae 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -24,5 +24,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index dcb1d6582..d9c0048e2 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -44,5 +44,5 @@ def build_completion(opt_parser): f.write(filled_template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 878b94166..811a3e9b5 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -3,6 +3,17 @@ import io import optparse +def read(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +# Get the version from yt_dlp/version.py without importing the package +def read_version(fname): + exec(compile(read(fname), fname, 'exec')) + return locals()['__version__'] + + def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') options, args = parser.parse_args() @@ -10,18 +21,9 @@ def main(): parser.error('Expected an input and an output filename') infile, outfile = args - - with open(infile, encoding='utf-8') as inf: - issue_template_tmpl = inf.read() - - # Get the version from yt_dlp/version.py without importing the package - exec(compile(open('yt_dlp/version.py').read(), - 'yt_dlp/version.py', 'exec')) - - out = issue_template_tmpl % {'version': locals()['__version__']} - with open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + outf.write( + read(infile) % {'version': read_version('yt_dlp/version.py')}) if __name__ == '__main__': diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5e2070602..6dc8fed90 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -21,7 +21,7 @@ from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor if os.path.exists(plugins_blocked_dirname): os.rename(plugins_blocked_dirname, plugins_dirname) -with open('devscripts/lazy_load_template.py') as f: +with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: module_template = f.read() CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 06660d8fd..59faea06a 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -43,5 +43,5 @@ def build_completion(opt_parser): f.write(template) -parser = yt_dlp.parseOpts()[0] +parser = yt_dlp.parseOpts(ignore_config_files=True)[0] build_completion(parser) -- cgit v1.2.3 From 1d485a1a799bbeeb2faea0595676ca7d4c0f3716 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 29 Apr 2022 07:18:36 +0530 Subject: [cleanup] Misc fixes Closes #3565, https://github.com/yt-dlp/yt-dlp/issues/3514#issuecomment-1105944364 --- devscripts/lazy_load_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index 0058915ae..e4b4f5825 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -7,7 +7,7 @@ class LazyLoadMetaClass(type): def __getattr__(cls, name): if '_real_class' not in cls.__dict__: write_string( - f'WARNING: Falling back to normal extractor since lazy extractor ' + 'WARNING: Falling back to normal extractor since lazy extractor ' f'{cls.__name__} does not have attribute {name}{bug_reports_message()}') return getattr(cls._get_real_class(), name) -- cgit v1.2.3 From 0f06bcd7591332937fdec497d6cbb4914358bc79 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 9 May 2022 17:24:28 +0530 Subject: [cleanup] Minor fixes (See desc) * [youtube] Fix `--youtube-skip-dash-manifest` * [build] Use `$()` in `Makefile`. Closes #3684 * Fix bug in 385ffb467b2285e85a2a5495b90314ba1f8e0700 * Fix bug in 43d7f5a5d0c77556156a3f8caa6976d3908a1e38 * [cleanup] Remove unnecessary `utf-8` from `str.encode`/`bytes.decode` * [utils] LazyList: Expose unnecessarily "protected" attributes and other minor cleanup --- devscripts/make_readme.py | 2 +- devscripts/make_supportedsites.py | 5 ++--- devscripts/update-formulae.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) (limited to 'devscripts') diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 1401c2e5a..fd234bf58 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -14,7 +14,7 @@ EPILOG_START = 'See full documentation' helptext = sys.stdin.read() if isinstance(helptext, bytes): - helptext = helptext.decode('utf-8') + helptext = helptext.decode() start, end = helptext.index(f'\n {OPTIONS_START}'), helptext.index(f'\n{EPILOG_START}') options = re.sub(r'(?m)^ (\w.+)$', r'## \1', helptext[start + 1: end + 1]) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 0a0d08f56..0403c1ae6 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -3,9 +3,8 @@ import optparse import os import sys -# Import yt_dlp -ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') -sys.path.insert(0, ROOT_DIR) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import yt_dlp diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py index 6424f5d9b..a89872c7b 100644 --- a/devscripts/update-formulae.py +++ b/devscripts/update-formulae.py @@ -17,7 +17,7 @@ normalized_version = '.'.join(str(int(x)) for x in version.split('.')) pypi_release = json.loads(compat_urllib_request.urlopen( 'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version -).read().decode('utf-8')) +).read().decode()) tarball_file = next(x for x in pypi_release['urls'] if x['filename'].endswith('.tar.gz')) -- cgit v1.2.3 From 8dcce6a89ca0dcb875fa1ba5f8a83cc244ceabcb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 9 May 2022 10:02:17 +0530 Subject: [extractor] Document netrc machines Closes #3169 --- devscripts/make_supportedsites.py | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) (limited to 'devscripts') diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 0403c1ae6..5531fec4d 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -5,38 +5,19 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import yt_dlp +from yt_dlp.extractor import list_extractors def main(): parser = optparse.OptionParser(usage='%prog OUTFILE.md') - options, args = parser.parse_args() + _, args = parser.parse_args() if len(args) != 1: parser.error('Expected an output filename') - outfile, = args - - def gen_ies_md(ies): - for ie in ies: - ie_md = f'**{ie.IE_NAME}**' - if ie.IE_DESC is False: - continue - if ie.IE_DESC is not None: - ie_md += f': {ie.IE_DESC}' - search_key = getattr(ie, 'SEARCH_KEY', None) - if search_key is not None: - ie_md += f'; "{ie.SEARCH_KEY}:" prefix' - if not ie.working(): - ie_md += ' (Currently broken)' - yield ie_md - - ies = sorted(yt_dlp.gen_extractors(), key=lambda i: i.IE_NAME.lower()) - out = '# Supported sites\n' + ''.join( - ' - ' + md + '\n' - for md in gen_ies_md(ies)) - - with open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + out = '\n'.join(ie.description() for ie in list_extractors(None) if ie.IE_DESC is not False) + + with open(args[0], 'w', encoding='utf-8') as outf: + outf.write(f'# Supported sites\n{out}\n') if __name__ == '__main__': -- cgit v1.2.3 From 82d020804de938bb7e87bd6bbc4961757b892cd0 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 11 May 2022 21:24:44 +0530 Subject: [extractor] Use classmethod/property where possible and refactor lazy extractors accordingly. This reduces the need to create extractor instances --- devscripts/lazy_load_template.py | 24 +++-- devscripts/make_lazy_extractors.py | 196 +++++++++++++++++++++---------------- devscripts/make_supportedsites.py | 4 +- 3 files changed, 123 insertions(+), 101 deletions(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index e4b4f5825..6d9b27742 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -1,30 +1,28 @@ +import importlib +import random import re -from ..utils import bug_reports_message, write_string +from ..utils import bug_reports_message, classproperty, write_string class LazyLoadMetaClass(type): def __getattr__(cls, name): - if '_real_class' not in cls.__dict__: + # "is_suitable" requires "_TESTS". However, they bloat the lazy_extractors + if '_real_class' not in cls.__dict__ and name not in ('is_suitable', 'get_testcases'): write_string( 'WARNING: Falling back to normal extractor since lazy extractor ' - f'{cls.__name__} does not have attribute {name}{bug_reports_message()}') - return getattr(cls._get_real_class(), name) + f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') + return getattr(cls.real_class, name) class LazyLoadExtractor(metaclass=LazyLoadMetaClass): - _module = None - _WORKING = True - - @classmethod - def _get_real_class(cls): + @classproperty + def real_class(cls): if '_real_class' not in cls.__dict__: - mod = __import__(cls._module, fromlist=(cls.__name__,)) - cls._real_class = getattr(mod, cls.__name__) + cls._real_class = getattr(importlib.import_module(cls._module), cls.__name__) return cls._real_class def __new__(cls, *args, **kwargs): - real_cls = cls._get_real_class() - instance = real_cls.__new__(real_cls) + instance = cls.real_class.__new__(cls.real_class) instance.__init__(*args, **kwargs) return instance diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 6dc8fed90..8ddc54b9b 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,101 +1,125 @@ #!/usr/bin/env python3 import os +import optparse import sys from inspect import getsource sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -lazy_extractors_filename = sys.argv[1] if len(sys.argv) > 1 else 'yt_dlp/extractor/lazy_extractors.py' -if os.path.exists(lazy_extractors_filename): - os.remove(lazy_extractors_filename) -# Block plugins from loading -plugins_dirname = 'ytdlp_plugins' -plugins_blocked_dirname = 'ytdlp_plugins_blocked' -if os.path.exists(plugins_dirname): - os.rename(plugins_dirname, plugins_blocked_dirname) - -from yt_dlp.extractor import _ALL_CLASSES -from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor - -if os.path.exists(plugins_blocked_dirname): - os.rename(plugins_blocked_dirname, plugins_dirname) - -with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: - module_template = f.read() - -CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id'] -module_contents = [ - module_template, - *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES], - '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] - -ie_template = ''' +NO_ATTR = object() +STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE'] +CLASS_METHODS = [ + 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', +] +IE_TEMPLATE = ''' class {name}({bases}): - _module = '{module}' + _module = {module!r} ''' - - -def get_base_name(base): - if base is InfoExtractor: - return 'LazyLoadExtractor' - elif base is SearchInfoExtractor: - return 'LazyLoadSearchExtractor' - else: - return base.__name__ - - -def build_lazy_ie(ie, name): - s = ie_template.format( - name=name, - bases=', '.join(map(get_base_name, ie.__bases__)), - module=ie.__module__) +with open('devscripts/lazy_load_template.py', encoding='utf-8') as f: + MODULE_TEMPLATE = f.read() + + +def main(): + parser = optparse.OptionParser(usage='%prog [OUTFILE.py]') + args = parser.parse_args()[1] or ['yt_dlp/extractor/lazy_extractors.py'] + if len(args) != 1: + parser.error('Expected only an output filename') + + lazy_extractors_filename = args[0] + if os.path.exists(lazy_extractors_filename): + os.remove(lazy_extractors_filename) + + _ALL_CLASSES = get_all_ies() # Must be before import + + from yt_dlp.extractor.common import InfoExtractor, SearchInfoExtractor + + DummyInfoExtractor = type('InfoExtractor', (InfoExtractor,), {'IE_NAME': NO_ATTR}) + module_src = '\n'.join(( + MODULE_TEMPLATE, + ' _module = None', + *extra_ie_code(DummyInfoExtractor), + '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', + *build_ies(_ALL_CLASSES, (InfoExtractor, SearchInfoExtractor), DummyInfoExtractor), + )) + + with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: + f.write(f'{module_src}\n') + + +def get_all_ies(): + PLUGINS_DIRNAME = 'ytdlp_plugins' + BLOCKED_DIRNAME = f'{PLUGINS_DIRNAME}_blocked' + if os.path.exists(PLUGINS_DIRNAME): + os.rename(PLUGINS_DIRNAME, BLOCKED_DIRNAME) + try: + from yt_dlp.extractor import _ALL_CLASSES + finally: + if os.path.exists(BLOCKED_DIRNAME): + os.rename(BLOCKED_DIRNAME, PLUGINS_DIRNAME) + return _ALL_CLASSES + + +def extra_ie_code(ie, base=None): + for var in STATIC_CLASS_PROPERTIES: + val = getattr(ie, var) + if val != (getattr(base, var) if base else NO_ATTR): + yield f' {var} = {val!r}' + yield '' + + for name in CLASS_METHODS: + f = getattr(ie, name) + if not base or f.__func__ != getattr(base, name).__func__: + yield getsource(f) + + +def build_ies(ies, bases, attr_base): + names = [] + for ie in sort_ies(ies, bases): + yield build_lazy_ie(ie, ie.__name__, attr_base) + if ie in ies: + names.append(ie.__name__) + + yield f'\n_ALL_CLASSES = [{", ".join(names)}]' + + +def sort_ies(ies, ignored_bases): + """find the correct sorting and add the required base classes so that subclasses can be correctly created""" + classes, returned_classes = ies[:-1], set() + assert ies[-1].__name__ == 'GenericIE', 'Last IE must be GenericIE' + while classes: + for c in classes[:]: + bases = set(c.__bases__) - {object, *ignored_bases} + restart = False + for b in bases: + if b not in classes and b not in returned_classes: + assert b.__name__ != 'GenericIE', 'Cannot inherit from GenericIE' + classes.insert(0, b) + restart = True + if restart: + break + if bases <= returned_classes: + yield c + returned_classes.add(c) + classes.remove(c) + break + yield ies[-1] + + +def build_lazy_ie(ie, name, attr_base): + bases = ', '.join({ + 'InfoExtractor': 'LazyLoadExtractor', + 'SearchInfoExtractor': 'LazyLoadSearchExtractor', + }.get(base.__name__, base.__name__) for base in ie.__bases__) + + s = IE_TEMPLATE.format(name=name, module=ie.__module__, bases=bases) valid_url = getattr(ie, '_VALID_URL', None) if not valid_url and hasattr(ie, '_make_valid_url'): valid_url = ie._make_valid_url() if valid_url: s += f' _VALID_URL = {valid_url!r}\n' - if not ie._WORKING: - s += ' _WORKING = False\n' - if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += f'\n{getsource(ie.suitable)}' - return s - - -# find the correct sorting and add the required base classes so that subclasses -# can be correctly created -classes = _ALL_CLASSES[:-1] -ordered_cls = [] -while classes: - for c in classes[:]: - bases = set(c.__bases__) - {object, InfoExtractor, SearchInfoExtractor} - stop = False - for b in bases: - if b not in classes and b not in ordered_cls: - if b.__name__ == 'GenericIE': - exit() - classes.insert(0, b) - stop = True - if stop: - break - if all(b in ordered_cls for b in bases): - ordered_cls.append(c) - classes.remove(c) - break -ordered_cls.append(_ALL_CLASSES[-1]) - -names = [] -for ie in ordered_cls: - name = ie.__name__ - src = build_lazy_ie(ie, name) - module_contents.append(src) - if ie in _ALL_CLASSES: - names.append(name) - -module_contents.append( - '\n_ALL_CLASSES = [{}]'.format(', '.join(names))) - -module_src = '\n'.join(module_contents) + '\n' - -with open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: - f.write(module_src) + return s + '\n'.join(extra_ie_code(ie, attr_base)) + + +if __name__ == '__main__': + main() diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 5531fec4d..d8c53c5e1 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -5,7 +5,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from yt_dlp.extractor import list_extractors +from yt_dlp.extractor import list_extractor_classes def main(): @@ -14,7 +14,7 @@ def main(): if len(args) != 1: parser.error('Expected an output filename') - out = '\n'.join(ie.description() for ie in list_extractors(None) if ie.IE_DESC is not False) + out = '\n'.join(ie.description() for ie in list_extractor_classes() if ie.IE_DESC is not False) with open(args[0], 'w', encoding='utf-8') as outf: outf.write(f'# Supported sites\n{out}\n') -- cgit v1.2.3 From 241464919271278831f23b3a086dcf57aeb80d3b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 16 May 2022 19:36:36 +0530 Subject: [cleanup] Misc cleanup --- devscripts/lazy_load_template.py | 11 ++++++++--- devscripts/make_lazy_extractors.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'devscripts') diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py index 6d9b27742..cdafaf1ef 100644 --- a/devscripts/lazy_load_template.py +++ b/devscripts/lazy_load_template.py @@ -2,13 +2,18 @@ import importlib import random import re -from ..utils import bug_reports_message, classproperty, write_string +from ..utils import ( + age_restricted, + bug_reports_message, + classproperty, + write_string, +) class LazyLoadMetaClass(type): def __getattr__(cls, name): - # "is_suitable" requires "_TESTS". However, they bloat the lazy_extractors - if '_real_class' not in cls.__dict__ and name not in ('is_suitable', 'get_testcases'): + # "_TESTS" bloat the lazy_extractors + if '_real_class' not in cls.__dict__ and name != 'get_testcases': write_string( 'WARNING: Falling back to normal extractor since lazy extractor ' f'{cls.__name__} does not have attribute {name}{bug_reports_message()}\n') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 8ddc54b9b..8c481bc2d 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -8,9 +8,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) NO_ATTR = object() -STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE'] +STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit'] CLASS_METHODS = [ - 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', + 'ie_key', 'working', 'description', 'suitable', '_match_valid_url', '_match_id', 'get_temp_id', 'is_suitable' ] IE_TEMPLATE = ''' class {name}({bases}): -- cgit v1.2.3