[compat] Split into sub-modules (#2173)

Authored by: fstirlitz, pukkandan

11 files changed, 274 insertions, 318 deletions

diff --git a/test/test_compat.py b/test/test_compat.py
index 29e7384f0..8e40a4180 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -42,14 +42,6 @@ class TestCompat(unittest.TestCase):
         finally:
             compat_setenv('HOME', old_home or '')
 
-    def test_all_present(self):
-        import yt_dlp.compat
-        all_names = yt_dlp.compat.__all__
-        present_names = set(filter(
-            lambda c: '_' in c and not c.startswith('_'),
-            dir(yt_dlp.compat))) - {'unicode_literals'}
-        self.assertEqual(all_names, sorted(present_names))
-
     def test_compat_urllib_parse_unquote(self):
         self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def')
         self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def')
diff --git a/yt_dlp/compat.py b/yt_dlp/compat.py
deleted file mode 100644
index f18c6cce2..000000000
--- a/yt_dlp/compat.py
+++ /dev/null
@@ -1,302 +0,0 @@
-import asyncio
-import base64
-import collections
-import contextlib
-import ctypes
-import getpass
-import html
-import html.parser
-import http
-import http.client
-import http.cookiejar
-import http.cookies
-import http.server
-import itertools
-import os
-import re
-import shlex
-import shutil
-import socket
-import struct
-import subprocess
-import sys
-import tokenize
-import urllib
-import xml.etree.ElementTree as etree
-from subprocess import DEVNULL
-
-
-# HTMLParseError has been deprecated in Python 3.3 and removed in
-# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
-# and uniform cross-version exception handling
-class compat_HTMLParseError(Exception):
-    pass
-
-
-# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
-# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines
-def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
-    return ctypes.WINFUNCTYPE(*args, **kwargs)
-
-
-class _TreeBuilder(etree.TreeBuilder):
-    def doctype(self, name, pubid, system):
-        pass
-
-
-def compat_etree_fromstring(text):
-    return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
-
-
-compat_os_name = os._name if os.name == 'java' else os.name
-
-
-if compat_os_name == 'nt':
-    def compat_shlex_quote(s):
-        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
-else:
-    from shlex import quote as compat_shlex_quote  # noqa: F401
-
-
-def compat_ord(c):
-    return c if isinstance(c, int) else ord(c)
-
-
-def compat_setenv(key, value, env=os.environ):
-    env[key] = value
-
-
-if compat_os_name == 'nt' and sys.version_info < (3, 8):
-    # os.path.realpath on Windows does not follow symbolic links
-    # prior to Python 3.8 (see https://bugs.python.org/issue9949)
-    def compat_realpath(path):
-        while os.path.islink(path):
-            path = os.path.abspath(os.readlink(path))
-        return path
-else:
-    compat_realpath = os.path.realpath
-
-
-try:
-    compat_Pattern = re.Pattern
-except AttributeError:
-    compat_Pattern = type(re.compile(''))
-
-
-try:
-    compat_Match = re.Match
-except AttributeError:
-    compat_Match = type(re.compile('').match(''))
-
-
-try:
-    compat_asyncio_run = asyncio.run  # >= 3.7
-except AttributeError:
-    def compat_asyncio_run(coro):
-        try:
-            loop = asyncio.get_event_loop()
-        except RuntimeError:
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-        loop.run_until_complete(coro)
-
-    asyncio.run = compat_asyncio_run
-
-
-try:  # >= 3.7
-    asyncio.tasks.all_tasks
-except AttributeError:
-    asyncio.tasks.all_tasks = asyncio.tasks.Task.all_tasks
-
-try:
-    import websockets as compat_websockets
-except ImportError:
-    compat_websockets = None
-
-# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
-# See https://github.com/yt-dlp/yt-dlp/issues/792
-# https://docs.python.org/3/library/os.path.html#os.path.expanduser
-if compat_os_name in ('nt', 'ce'):
-    def compat_expanduser(path):
-        HOME = os.environ.get('HOME')
-        if not HOME:
-            return os.path.expanduser(path)
-        elif not path.startswith('~'):
-            return path
-        i = path.replace('\\', '/', 1).find('/')  # ~user
-        if i < 0:
-            i = len(path)
-        userhome = os.path.join(os.path.dirname(HOME), path[1:i]) if i > 1 else HOME
-        return userhome + path[i:]
-else:
-    compat_expanduser = os.path.expanduser
-
-
-try:
-    from Cryptodome.Cipher import AES as compat_pycrypto_AES
-except ImportError:
-    try:
-        from Crypto.Cipher import AES as compat_pycrypto_AES
-    except ImportError:
-        compat_pycrypto_AES = None
-
-try:
-    import brotlicffi as compat_brotli
-except ImportError:
-    try:
-        import brotli as compat_brotli
-    except ImportError:
-        compat_brotli = None
-
-WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
-
-
-def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
-    if compat_os_name != 'nt':
-        return
-    global WINDOWS_VT_MODE
-    startupinfo = subprocess.STARTUPINFO()
-    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
-    with contextlib.suppress(Exception):
-        subprocess.Popen('', shell=True, startupinfo=startupinfo).wait()
-        WINDOWS_VT_MODE = True
-
-
-#  Deprecated
-
-compat_b64decode = base64.b64decode
-compat_chr = chr
-compat_cookiejar = http.cookiejar
-compat_cookiejar_Cookie = http.cookiejar.Cookie
-compat_cookies_SimpleCookie = http.cookies.SimpleCookie
-compat_get_terminal_size = shutil.get_terminal_size
-compat_getenv = os.getenv
-compat_getpass = getpass.getpass
-compat_html_entities = html.entities
-compat_html_entities_html5 = html.entities.html5
-compat_HTMLParser = html.parser.HTMLParser
-compat_http_client = http.client
-compat_http_server = http.server
-compat_HTTPError = urllib.error.HTTPError
-compat_itertools_count = itertools.count
-compat_parse_qs = urllib.parse.parse_qs
-compat_str = str
-compat_struct_pack = struct.pack
-compat_struct_unpack = struct.unpack
-compat_tokenize_tokenize = tokenize.tokenize
-compat_urllib_error = urllib.error
-compat_urllib_parse_unquote = urllib.parse.unquote
-compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
-compat_urllib_parse_urlencode = urllib.parse.urlencode
-compat_urllib_parse_urlparse = urllib.parse.urlparse
-compat_urllib_request = urllib.request
-compat_urlparse = compat_urllib_parse = urllib.parse
-
-
-# To be removed - Do not use
-
-compat_basestring = str
-compat_collections_abc = collections.abc
-compat_cookies = http.cookies
-compat_etree_Element = etree.Element
-compat_etree_register_namespace = etree.register_namespace
-compat_filter = filter
-compat_input = input
-compat_integer_types = (int, )
-compat_kwargs = lambda kwargs: kwargs
-compat_map = map
-compat_numeric_types = (int, float, complex)
-compat_print = print
-compat_shlex_split = shlex.split
-compat_socket_create_connection = socket.create_connection
-compat_Struct = struct.Struct
-compat_subprocess_get_DEVNULL = lambda: DEVNULL
-compat_urllib_parse_quote = urllib.parse.quote
-compat_urllib_parse_quote_plus = urllib.parse.quote_plus
-compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
-compat_urllib_parse_urlunparse = urllib.parse.urlunparse
-compat_urllib_request_DataHandler = urllib.request.DataHandler
-compat_urllib_response = urllib.response
-compat_urlretrieve = urllib.request.urlretrieve
-compat_xml_parse_error = etree.ParseError
-compat_xpath = lambda xpath: xpath
-compat_zip = zip
-workaround_optparse_bug9161 = lambda: None
-
-
-# Set public objects
-
-__all__ = [
-    'WINDOWS_VT_MODE',
-    'compat_HTMLParseError',
-    'compat_HTMLParser',
-    'compat_HTTPError',
-    'compat_Match',
-    'compat_Pattern',
-    'compat_Struct',
-    'compat_asyncio_run',
-    'compat_b64decode',
-    'compat_basestring',
-    'compat_brotli',
-    'compat_chr',
-    'compat_collections_abc',
-    'compat_cookiejar',
-    'compat_cookiejar_Cookie',
-    'compat_cookies',
-    'compat_cookies_SimpleCookie',
-    'compat_ctypes_WINFUNCTYPE',
-    'compat_etree_Element',
-    'compat_etree_fromstring',
-    'compat_etree_register_namespace',
-    'compat_expanduser',
-    'compat_filter',
-    'compat_get_terminal_size',
-    'compat_getenv',
-    'compat_getpass',
-    'compat_html_entities',
-    'compat_html_entities_html5',
-    'compat_http_client',
-    'compat_http_server',
-    'compat_input',
-    'compat_integer_types',
-    'compat_itertools_count',
-    'compat_kwargs',
-    'compat_map',
-    'compat_numeric_types',
-    'compat_ord',
-    'compat_os_name',
-    'compat_parse_qs',
-    'compat_print',
-    'compat_pycrypto_AES',
-    'compat_realpath',
-    'compat_setenv',
-    'compat_shlex_quote',
-    'compat_shlex_split',
-    'compat_socket_create_connection',
-    'compat_str',
-    'compat_struct_pack',
-    'compat_struct_unpack',
-    'compat_subprocess_get_DEVNULL',
-    'compat_tokenize_tokenize',
-    'compat_urllib_error',
-    'compat_urllib_parse',
-    'compat_urllib_parse_quote',
-    'compat_urllib_parse_quote_plus',
-    'compat_urllib_parse_unquote',
-    'compat_urllib_parse_unquote_plus',
-    'compat_urllib_parse_unquote_to_bytes',
-    'compat_urllib_parse_urlencode',
-    'compat_urllib_parse_urlparse',
-    'compat_urllib_parse_urlunparse',
-    'compat_urllib_request',
-    'compat_urllib_request_DataHandler',
-    'compat_urllib_response',
-    'compat_urlparse',
-    'compat_urlretrieve',
-    'compat_websockets',
-    'compat_xml_parse_error',
-    'compat_xpath',
-    'compat_zip',
-    'windows_enable_vt_mode',
-    'workaround_optparse_bug9161',
-]
diff --git a/yt_dlp/compat/__init__.py b/yt_dlp/compat/__init__.py
new file mode 100644
index 000000000..7a0e82992
--- /dev/null
+++ b/yt_dlp/compat/__init__.py
@@ -0,0 +1,129 @@
+import contextlib
+import os
+import subprocess
+import sys
+import types
+import xml.etree.ElementTree as etree
+
+from . import re
+from ._deprecated import *  # noqa: F401, F403
+
+
+# HTMLParseError has been deprecated in Python 3.3 and removed in
+# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
+# and uniform cross-version exception handling
+class compat_HTMLParseError(Exception):
+    pass
+
+
+class _TreeBuilder(etree.TreeBuilder):
+    def doctype(self, name, pubid, system):
+        pass
+
+
+def compat_etree_fromstring(text):
+    return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
+
+
+compat_os_name = os._name if os.name == 'java' else os.name
+
+
+if compat_os_name == 'nt':
+    def compat_shlex_quote(s):
+        return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
+else:
+    from shlex import quote as compat_shlex_quote  # noqa: F401
+
+
+def compat_ord(c):
+    return c if isinstance(c, int) else ord(c)
+
+
+def compat_setenv(key, value, env=os.environ):
+    env[key] = value
+
+
+if compat_os_name == 'nt' and sys.version_info < (3, 8):
+    # os.path.realpath on Windows does not follow symbolic links
+    # prior to Python 3.8 (see https://bugs.python.org/issue9949)
+    def compat_realpath(path):
+        while os.path.islink(path):
+            path = os.path.abspath(os.readlink(path))
+        return path
+else:
+    compat_realpath = os.path.realpath
+
+
+try:
+    import websockets as compat_websockets
+except ImportError:
+    compat_websockets = None
+
+# Python 3.8+ does not honor %HOME% on windows, but this breaks compatibility with youtube-dl
+# See https://github.com/yt-dlp/yt-dlp/issues/792
+# https://docs.python.org/3/library/os.path.html#os.path.expanduser
+if compat_os_name in ('nt', 'ce'):
+    def compat_expanduser(path):
+        HOME = os.environ.get('HOME')
+        if not HOME:
+            return os.path.expanduser(path)
+        elif not path.startswith('~'):
+            return path
+        i = path.replace('\\', '/', 1).find('/')  # ~user
+        if i < 0:
+            i = len(path)
+        userhome = os.path.join(os.path.dirname(HOME), path[1:i]) if i > 1 else HOME
+        return userhome + path[i:]
+else:
+    compat_expanduser = os.path.expanduser
+
+
+try:
+    from Cryptodome.Cipher import AES as compat_pycrypto_AES
+except ImportError:
+    try:
+        from Crypto.Cipher import AES as compat_pycrypto_AES
+    except ImportError:
+        compat_pycrypto_AES = None
+
+try:
+    import brotlicffi as compat_brotli
+except ImportError:
+    try:
+        import brotli as compat_brotli
+    except ImportError:
+        compat_brotli = None
+
+WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
+
+
+def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
+    if compat_os_name != 'nt':
+        return
+    global WINDOWS_VT_MODE
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+    with contextlib.suppress(Exception):
+        subprocess.Popen('', shell=True, startupinfo=startupinfo).wait()
+        WINDOWS_VT_MODE = True
+
+
+class _PassthroughLegacy(types.ModuleType):
+    def __getattr__(self, attr):
+        import importlib
+        with contextlib.suppress(ImportError):
+            return importlib.import_module(f'.{attr}', __name__)
+
+        legacy = importlib.import_module('._legacy', __name__)
+        if not hasattr(legacy, attr):
+            raise AttributeError(f'module {__name__} has no attribute {attr}')
+
+        # XXX: Implement this the same way as other DeprecationWarnings without circular import
+        import warnings
+        warnings.warn(DeprecationWarning(f'{__name__}.{attr} is deprecated'), stacklevel=2)
+        return getattr(legacy, attr)
+
+
+# Python 3.6 does not have module level __getattr__
+# https://peps.python.org/pep-0562/
+sys.modules[__name__].__class__ = _PassthroughLegacy
diff --git a/yt_dlp/compat/_deprecated.py b/yt_dlp/compat/_deprecated.py
new file mode 100644
index 000000000..f84439825
--- /dev/null
+++ b/yt_dlp/compat/_deprecated.py
@@ -0,0 +1,47 @@
+"""Deprecated - New code should avoid these"""
+
+import base64
+import getpass
+import html
+import html.parser
+import http
+import http.client
+import http.cookiejar
+import http.cookies
+import http.server
+import itertools
+import os
+import shutil
+import struct
+import tokenize
+import urllib
+
+compat_b64decode = base64.b64decode
+compat_chr = chr
+compat_cookiejar = http.cookiejar
+compat_cookiejar_Cookie = http.cookiejar.Cookie
+compat_cookies_SimpleCookie = http.cookies.SimpleCookie
+compat_get_terminal_size = shutil.get_terminal_size
+compat_getenv = os.getenv
+compat_getpass = getpass.getpass
+compat_html_entities = html.entities
+compat_html_entities_html5 = html.entities.html5
+compat_HTMLParser = html.parser.HTMLParser
+compat_http_client = http.client
+compat_http_server = http.server
+compat_HTTPError = urllib.error.HTTPError
+compat_itertools_count = itertools.count
+compat_parse_qs = urllib.parse.parse_qs
+compat_str = str
+compat_struct_pack = struct.pack
+compat_struct_unpack = struct.unpack
+compat_tokenize_tokenize = tokenize.tokenize
+compat_urllib_error = urllib.error
+compat_urllib_parse_unquote = urllib.parse.unquote
+compat_urllib_parse_unquote_plus = urllib.parse.unquote_plus
+compat_urllib_parse_urlencode = urllib.parse.urlencode
+compat_urllib_parse_urlparse = urllib.parse.urlparse
+compat_urllib_request = urllib.request
+compat_urlparse = compat_urllib_parse = urllib.parse
+
+__all__ = [x for x in globals() if x.startswith('compat_')]
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
new file mode 100644
index 000000000..f185b7e2f
--- /dev/null
+++ b/yt_dlp/compat/_legacy.py
@@ -0,0 +1,54 @@
+""" Do not use! """
+
+import collections
+import ctypes
+import http
+import http.client
+import http.cookiejar
+import http.cookies
+import http.server
+import shlex
+import socket
+import struct
+import urllib
+import xml.etree.ElementTree as etree
+from subprocess import DEVNULL
+
+from .asyncio import run as compat_asyncio_run  # noqa: F401
+from .re import Pattern as compat_Pattern  # noqa: F401
+from .re import match as compat_Match  # noqa: F401
+
+
+# compat_ctypes_WINFUNCTYPE = ctypes.WINFUNCTYPE
+# will not work since ctypes.WINFUNCTYPE does not exist in UNIX machines
+def compat_ctypes_WINFUNCTYPE(*args, **kwargs):
+    return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
+compat_basestring = str
+compat_collections_abc = collections.abc
+compat_cookies = http.cookies
+compat_etree_Element = etree.Element
+compat_etree_register_namespace = etree.register_namespace
+compat_filter = filter
+compat_input = input
+compat_integer_types = (int, )
+compat_kwargs = lambda kwargs: kwargs
+compat_map = map
+compat_numeric_types = (int, float, complex)
+compat_print = print
+compat_shlex_split = shlex.split
+compat_socket_create_connection = socket.create_connection
+compat_Struct = struct.Struct
+compat_subprocess_get_DEVNULL = lambda: DEVNULL
+compat_urllib_parse_quote = urllib.parse.quote
+compat_urllib_parse_quote_plus = urllib.parse.quote_plus
+compat_urllib_parse_unquote_to_bytes = urllib.parse.unquote_to_bytes
+compat_urllib_parse_urlunparse = urllib.parse.urlunparse
+compat_urllib_request_DataHandler = urllib.request.DataHandler
+compat_urllib_response = urllib.response
+compat_urlretrieve = urllib.request.urlretrieve
+compat_xml_parse_error = etree.ParseError
+compat_xpath = lambda xpath: xpath
+compat_zip = zip
+workaround_optparse_bug9161 = lambda: None
diff --git a/yt_dlp/compat/asyncio/__init__.py b/yt_dlp/compat/asyncio/__init__.py
new file mode 100644
index 000000000..0e8c6cad3
--- /dev/null
+++ b/yt_dlp/compat/asyncio/__init__.py
@@ -0,0 +1,16 @@
+# flake8: noqa: F405
+
+from asyncio import *  # noqa: F403
+
+from . import tasks  # noqa: F401
+
+try:
+    run  # >= 3.7
+except NameError:
+    def run(coro):
+        try:
+            loop = get_event_loop()
+        except RuntimeError:
+            loop = new_event_loop()
+            set_event_loop(loop)
+        loop.run_until_complete(coro)
diff --git a/yt_dlp/compat/asyncio/tasks.py b/yt_dlp/compat/asyncio/tasks.py
new file mode 100644
index 000000000..cb31e52fa
--- /dev/null
+++ b/yt_dlp/compat/asyncio/tasks.py
@@ -0,0 +1,8 @@
+# flake8: noqa: F405
+
+from asyncio.tasks import *  # noqa: F403
+
+try:  # >= 3.7
+    all_tasks
+except NameError:
+    all_tasks = Task.all_tasks
diff --git a/yt_dlp/compat/re.py b/yt_dlp/compat/re.py
new file mode 100644
index 000000000..e8a6fabbd
--- /dev/null
+++ b/yt_dlp/compat/re.py
@@ -0,0 +1,14 @@
+# flake8: noqa: F405
+
+from re import *  # F403
+
+try:
+    Pattern  # >= 3.7
+except NameError:
+    Pattern = type(compile(''))
+
+
+try:
+    Match  # >= 3.7
+except NameError:
+    Match = type(compile('').match(''))
diff --git a/yt_dlp/downloader/websocket.py b/yt_dlp/downloader/websocket.py
index 6b190cd90..8465f9713 100644
--- a/yt_dlp/downloader/websocket.py
+++ b/yt_dlp/downloader/websocket.py
@@ -1,4 +1,3 @@
-import asyncio
 import contextlib
 import os
 import signal
@@ -15,6 +14,7 @@ else:
 
 from .common import FileDownloader
 from .external import FFmpegFD
+from ..compat import asyncio
 
 
 class FFmpegSinkFD(FileDownloader):
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 10b297708..3ee5e257c 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -7,7 +7,6 @@ import math
 import netrc
 import os
 import random
-import re
 import sys
 import time
 import xml.etree.ElementTree
@@ -20,13 +19,13 @@ from ..compat import (
     compat_getpass,
     compat_http_client,
     compat_os_name,
-    compat_Pattern,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
     compat_urllib_parse_urlencode,
     compat_urllib_request,
     compat_urlparse,
+    re,
 )
 from ..downloader import FileDownloader
 from ..downloader.f4m import get_base_url, remove_encrypted_media
@@ -1198,7 +1197,7 @@ class InfoExtractor:
         """
         if string is None:
             mobj = None
-        elif isinstance(pattern, (str, compat_Pattern)):
+        elif isinstance(pattern, (str, re.Pattern)):
             mobj = re.search(pattern, string, flags)
         else:
             for p in pattern:
diff --git a/yt_dlp/webvtt.py b/yt_dlp/webvtt.py
index 741622b25..4c222ba8e 100644
--- a/yt_dlp/webvtt.py
+++ b/yt_dlp/webvtt.py
@@ -9,9 +9,8 @@ in RFC 8216 §3.5 <https://tools.ietf.org/html/rfc8216#section-3.5>.
 """
 
 import io
-import re
 
-from .compat import compat_Match, compat_Pattern
+from .compat import re
 from .utils import int_or_none, timetuple_from_msec
 
 
@@ -26,7 +25,7 @@ class _MatchParser:
         self._pos = 0
 
     def match(self, r):
-        if isinstance(r, compat_Pattern):
+        if isinstance(r, re.Pattern):
             return r.match(self._data, self._pos)
         if isinstance(r, str):
             if self._data.startswith(r, self._pos):
@@ -37,7 +36,7 @@ class _MatchParser:
     def advance(self, by):
         if by is None:
             amt = 0
-        elif isinstance(by, compat_Match):
+        elif isinstance(by, re.Match):
             amt = len(by.group(0))
         elif isinstance(by, str):
             amt = len(by)