From 591bb9d3553a4d7b453777c1e28e0948741e3b50 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 17 May 2022 18:36:29 +0530
Subject: Fix color in `-q -F`

and convert `ydl._out_files`/`ydl._allow_colors` to `Namespace`

Closes #3761
---
 yt_dlp/utils.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 8a9567de4..1249c0100 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5322,8 +5322,20 @@ class classproperty:
         return self.f(cls)
 
 
-def Namespace(**kwargs):
-    return collections.namedtuple('Namespace', kwargs)(**kwargs)
+class Namespace:
+    """Immutable namespace"""
+    @property
+    def items_(self):
+        return self._dict.items()
+
+    def __init__(self, **kwargs):
+        self._dict = kwargs
+
+    def __getattr__(self, attr):
+        return self._dict[attr]
+
+    def __repr__(self):
+        return f'{type(self).__name__}({", ".join(f"{k}={v}" for k, v in self.items_)})'
 
 
 # Deprecated
-- 
cgit v1.2.3


From 7896214c42db91bbf62853b5c7359c9e83064cf1 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 17 May 2022 22:08:12 +0530
Subject: Bugfix for 591bb9d3553a4d7b453777c1e28e0948741e3b50

Closes #3769
---
 yt_dlp/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 1249c0100..48a94415d 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5324,9 +5324,6 @@ class classproperty:
 
 class Namespace:
     """Immutable namespace"""
-    @property
-    def items_(self):
-        return self._dict.items()
 
     def __init__(self, **kwargs):
         self._dict = kwargs
@@ -5334,8 +5331,14 @@ class Namespace:
     def __getattr__(self, attr):
         return self._dict[attr]
 
+    def __contains__(self, item):
+        return item in self._dict.values()
+
+    def __iter__(self):
+        return iter(self._dict.items())
+
     def __repr__(self):
-        return f'{type(self).__name__}({", ".join(f"{k}={v}" for k, v in self.items_)})'
+        return f'{type(self).__name__}({", ".join(f"{k}={v}" for k, v in self)})'
 
 
 # Deprecated
-- 
cgit v1.2.3


From 80e8493ee7c3083f4e215794e4a67ba5265f24f7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 18 May 2022 06:42:43 +0530
Subject: [utils] `is_html`: Handle double BOM

Closes #2885
---
 yt_dlp/utils.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 48a94415d..3b0e6750c 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3290,14 +3290,13 @@ def is_html(first_bytes):
         (b'\xff\xfe', 'utf-16-le'),
         (b'\xfe\xff', 'utf-16-be'),
     ]
+
+    encoding = 'utf-8'
     for bom, enc in BOMS:
-        if first_bytes.startswith(bom):
-            s = first_bytes[len(bom):].decode(enc, 'replace')
-            break
-    else:
-        s = first_bytes.decode('utf-8', 'replace')
+        while first_bytes.startswith(bom):
+            encoding, first_bytes = enc, first_bytes[len(bom):]
 
-    return re.match(r'^\s*<', s)
+    return re.match(r'^\s*<', first_bytes.decode(encoding, 'replace'))
 
 
 def determine_protocol(info_dict):
-- 
cgit v1.2.3


From 21633673c33f082c6673bc245e4a90d880729a58 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 18 May 2022 09:04:30 +0530
Subject: [cleanup] Minor fixes

---
 yt_dlp/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 3b0e6750c..bcdb7d55b 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -714,7 +714,9 @@ def sanitize_path(s, force=False):
 def sanitize_url(url):
     # Prepend protocol-less URLs with `http:` scheme in order to mitigate
     # the number of unwanted failures due to missing protocol
-    if url.startswith('//'):
+    if url is None:
+        return
+    elif url.startswith('//'):
         return 'http:%s' % url
     # Fix some common typos seen so far
     COMMON_TYPOS = (
-- 
cgit v1.2.3


From 9e491463521c65ca4d1d44a757e0a115f62834f5 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 19 May 2022 19:45:21 +0530
Subject: Add option `--alias`

---
 yt_dlp/utils.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index bcdb7d55b..f02f71177 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5153,11 +5153,12 @@ def parse_http_range(range):
 
 class Config:
     own_args = None
+    parsed_args = None
     filename = None
     __initialized = False
 
     def __init__(self, parser, label=None):
-        self._parser, self.label = parser, label
+        self.parser, self.label = parser, label
         self._loaded_paths, self.configs = set(), []
 
     def init(self, args=None, filename=None):
@@ -5170,14 +5171,16 @@ class Config:
                 return False
             self._loaded_paths.add(location)
 
-        self.__initialized = True
-        self.own_args, self.filename = args, filename
-        for location in self._parser.parse_args(args)[0].config_locations or []:
+        self.own_args, self.__initialized = args, True
+        opts, _ = self.parser.parse_known_args(args)
+        self.parsed_args, self.filename = args, filename
+
+        for location in opts.config_locations or []:
             location = os.path.join(directory, expand_path(location))
             if os.path.isdir(location):
                 location = os.path.join(location, 'yt-dlp.conf')
             if not os.path.exists(location):
-                self._parser.error(f'config location {location} does not exist')
+                self.parser.error(f'config location {location} does not exist')
             self.append_config(self.read_file(location), location)
         return True
 
@@ -5223,7 +5226,7 @@ class Config:
         return opts
 
     def append_config(self, *args, label=None):
-        config = type(self)(self._parser, label)
+        config = type(self)(self.parser, label)
         config._loaded_paths = self._loaded_paths
         if config.init(*args):
             self.configs.append(config)
@@ -5232,10 +5235,13 @@ class Config:
     def all_args(self):
         for config in reversed(self.configs):
             yield from config.all_args
-        yield from self.own_args or []
+        yield from self.parsed_args or []
+
+    def parse_known_args(self, **kwargs):
+        return self.parser.parse_known_args(self.all_args, **kwargs)
 
     def parse_args(self):
-        return self._parser.parse_args(self.all_args)
+        return self.parser.parse_args(self.all_args)
 
 
 class WebSocketsWrapper():
-- 
cgit v1.2.3


From 2f97cc615bdb788bf5c86c1132144ca491b820c3 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 18 May 2022 14:06:41 +0530
Subject: [utils] `ISO3166Utils`: Add `EU` and `AP`

Fixes https://github.com/yt-dlp/yt-dlp/pull/3302#discussion_r875528517
---
 yt_dlp/utils.py | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index f02f71177..41157f5de 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -4166,6 +4166,9 @@ class ISO3166Utils:
         'YE': 'Yemen',
         'ZM': 'Zambia',
         'ZW': 'Zimbabwe',
+        # Not ISO 3166 codes, but used for IP blocks
+        'AP': 'Asia/Pacific Region',
+        'EU': 'Europe',
     }
 
     @classmethod
-- 
cgit v1.2.3


From 0b9c08b47bb5e95c21b067044ace4e824d19a9c2 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 19 May 2022 19:36:31 +0530
Subject: [utils] Improve performance using `functools.cache`

Closes #3786
---
 yt_dlp/utils.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 41157f5de..0274e330d 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -11,7 +11,6 @@ import datetime
 import email.header
 import email.utils
 import errno
-import functools
 import gzip
 import hashlib
 import hmac
@@ -39,8 +38,8 @@ import urllib.parse
 import xml.etree.ElementTree
 import zlib
 
+from .compat import asyncio, functools  # Modules
 from .compat import (
-    asyncio,
     compat_chr,
     compat_cookiejar,
     compat_etree_fromstring,
@@ -248,6 +247,7 @@ JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<jso
 NUMBER_RE = r'\d+(?:\.\d+)?'
 
 
+@functools.cache
 def preferredencoding():
     """Get preferred encoding.
 
@@ -1883,6 +1883,7 @@ def platform_name():
     return res
 
 
+@functools.cache
 def get_windows_version():
     ''' Get Windows version. None if it's not running on Windows '''
     if compat_os_name == 'nt':
@@ -2079,6 +2080,7 @@ class locked_file:
         return iter(self.f)
 
 
+@functools.cache
 def get_filesystem_encoding():
     encoding = sys.getfilesystemencoding()
     return encoding if encoding is not None else 'utf-8'
@@ -5092,6 +5094,7 @@ def jwt_decode_hs256(jwt):
     return payload_data
 
 
+@functools.cache
 def supports_terminal_sequences(stream):
     if compat_os_name == 'nt':
         from .compat import WINDOWS_VT_MODE  # Must be imported locally
-- 
cgit v1.2.3


From 53973b4d2cb349d39d6f240911142b330d1dd80d Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 20 May 2022 03:02:25 +0530
Subject: [utils] Fix bug in 0b9c08b47bb5e95c21b067044ace4e824d19a9c2

* Cache of `supports_terminal_sequences` must be reset after enabling VT mode
* and move `windows_enable_vt_mode` to utils to avoid cyclic imports
---
 yt_dlp/utils.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 0274e330d..78789b1c5 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5094,10 +5094,12 @@ def jwt_decode_hs256(jwt):
     return payload_data
 
 
+WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
+
+
 @functools.cache
 def supports_terminal_sequences(stream):
     if compat_os_name == 'nt':
-        from .compat import WINDOWS_VT_MODE  # Must be imported locally
         if not WINDOWS_VT_MODE or get_windows_version() < (10, 0, 10586):
             return False
     elif not os.getenv('TERM'):
@@ -5108,6 +5110,21 @@ def supports_terminal_sequences(stream):
         return False
 
 
+def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
+    if compat_os_name != 'nt':
+        return
+    global WINDOWS_VT_MODE
+    startupinfo = subprocess.STARTUPINFO()
+    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+    try:
+        subprocess.Popen('', shell=True, startupinfo=startupinfo).wait()
+    except Exception:
+        return
+
+    WINDOWS_VT_MODE = True
+    supports_terminal_sequences.cache_clear()
+
+
 _terminal_sequences_re = re.compile('\033\\[[^m]+m')
 
 
-- 
cgit v1.2.3


From c487cf00101525ff836d59a2a42ef63e85ea9556 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 17 Apr 2022 22:48:50 +0530
Subject: [cleanup] Misc

---
 yt_dlp/utils.py | 73 +++++++++++++++++++++++++--------------------------------
 1 file changed, 32 insertions(+), 41 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 78789b1c5..12204433d 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -38,7 +38,7 @@ import urllib.parse
 import xml.etree.ElementTree
 import zlib
 
-from .compat import asyncio, functools  # Modules
+from .compat import asyncio, functools  # isort: split
 from .compat import (
     compat_chr,
     compat_cookiejar,
@@ -362,14 +362,14 @@ def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
     return n.attrib[key]
 
 
-def get_element_by_id(id, html):
+def get_element_by_id(id, html, **kwargs):
     """Return the content of the tag with the specified ID in the passed HTML document"""
-    return get_element_by_attribute('id', id, html)
+    return get_element_by_attribute('id', id, html, **kwargs)
 
 
-def get_element_html_by_id(id, html):
+def get_element_html_by_id(id, html, **kwargs):
     """Return the html of the tag with the specified ID in the passed HTML document"""
-    return get_element_html_by_attribute('id', id, html)
+    return get_element_html_by_attribute('id', id, html, **kwargs)
 
 
 def get_element_by_class(class_name, html):
@@ -384,17 +384,17 @@ def get_element_html_by_class(class_name, html):
     return retval[0] if retval else None
 
 
-def get_element_by_attribute(attribute, value, html, escape_value=True):
-    retval = get_elements_by_attribute(attribute, value, html, escape_value)
+def get_element_by_attribute(attribute, value, html, **kwargs):
+    retval = get_elements_by_attribute(attribute, value, html, **kwargs)
     return retval[0] if retval else None
 
 
-def get_element_html_by_attribute(attribute, value, html, escape_value=True):
-    retval = get_elements_html_by_attribute(attribute, value, html, escape_value)
+def get_element_html_by_attribute(attribute, value, html, **kargs):
+    retval = get_elements_html_by_attribute(attribute, value, html, **kargs)
     return retval[0] if retval else None
 
 
-def get_elements_by_class(class_name, html):
+def get_elements_by_class(class_name, html, **kargs):
     """Return the content of all tags with the specified class in the passed HTML document as a list"""
     return get_elements_by_attribute(
         'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
@@ -1899,15 +1899,14 @@ def write_string(s, out=None, encoding=None):
     if compat_os_name == 'nt' and supports_terminal_sequences(out):
         s = re.sub(r'([\r\n]+)', r' \1', s)
 
+    enc = None
     if 'b' in getattr(out, 'mode', ''):
-        byt = s.encode(encoding or preferredencoding(), 'ignore')
-        out.write(byt)
+        enc = encoding or preferredencoding()
     elif hasattr(out, 'buffer'):
+        out = out.buffer
         enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
-        byt = s.encode(enc, 'ignore')
-        out.buffer.write(byt)
-    else:
-        out.write(s)
+
+    out.write(s.encode(enc, 'ignore') if enc else s)
     out.flush()
 
 
@@ -2970,7 +2969,7 @@ TV_PARENTAL_GUIDELINES = {
 
 def parse_age_limit(s):
     # isinstance(False, int) is True. So type() must be used instead
-    if type(s) is int:
+    if type(s) is int:  # noqa: E721
         return s if 0 <= s <= 21 else None
     elif not isinstance(s, str):
         return None
@@ -3656,26 +3655,21 @@ def dfxp2srt(dfxp_data):
     return ''.join(out)
 
 
-def cli_option(params, command_option, param):
+def cli_option(params, command_option, param, separator=None):
     param = params.get(param)
-    if param:
-        param = compat_str(param)
-    return [command_option, param] if param is not None else []
+    return ([] if param is None
+            else [command_option, str(param)] if separator is None
+            else [f'{command_option}{separator}{param}'])
 
 
 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
     param = params.get(param)
-    if param is None:
-        return []
-    assert isinstance(param, bool)
-    if separator:
-        return [command_option + separator + (true_value if param else false_value)]
-    return [command_option, true_value if param else false_value]
+    assert param in (True, False, None)
+    return cli_option({True: true_value, False: false_value}, command_option, param, separator)
 
 
 def cli_valueless_option(params, command_option, param, expected_value=True):
-    param = params.get(param)
-    return [command_option] if param == expected_value else []
+    return [command_option] if params.get(param) == expected_value else []
 
 
 def cli_configuration_args(argdict, keys, default=[], use_compat=True):
@@ -4910,14 +4904,9 @@ def make_dir(path, to_screen=None):
 
 
 def get_executable_path():
-    from zipimport import zipimporter
-    if hasattr(sys, 'frozen'):  # Running from PyInstaller
-        path = os.path.dirname(sys.executable)
-    elif isinstance(__loader__, zipimporter):  # Running from ZIP
-        path = os.path.join(os.path.dirname(__file__), '../..')
-    else:
-        path = os.path.join(os.path.dirname(__file__), '..')
-    return os.path.abspath(path)
+    from .update import get_variant_and_executable_path
+
+    return os.path.abspath(get_variant_and_executable_path()[1])
 
 
 def load_plugins(name, suffix, namespace):
@@ -5344,12 +5333,14 @@ def merge_headers(*dicts):
 
 
 class classproperty:
-    def __init__(self, f):
-        functools.update_wrapper(self, f)
-        self.f = f
+    """classmethod(property(func)) that works in py < 3.9"""
+
+    def __init__(self, func):
+        functools.update_wrapper(self, func)
+        self.func = func
 
     def __get__(self, _, cls):
-        return self.f(cls)
+        return self.func(cls)
 
 
 class Namespace:
-- 
cgit v1.2.3


From b5899f4f19116bb4d98907413fa3fb84a952ef13 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 22 May 2022 17:07:18 +0530
Subject: [build, cleanup] Refactor

Closes #3835, #3837
---
 yt_dlp/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 12204433d..2e3c51562 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -4904,9 +4904,9 @@ def make_dir(path, to_screen=None):
 
 
 def get_executable_path():
-    from .update import get_variant_and_executable_path
+    from .update import _get_variant_and_executable_path
 
-    return os.path.abspath(get_variant_and_executable_path()[1])
+    return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1]))
 
 
 def load_plugins(name, suffix, namespace):
-- 
cgit v1.2.3


From 6b9e832db7dedcd6f2e7be1bf44f56a91ff18737 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 24 May 2022 17:30:28 +0530
Subject: `--config-location -` to provide options interactively

---
 yt_dlp/utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 2e3c51562..6701492f2 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5163,6 +5163,12 @@ def parse_http_range(range):
     return int(crg.group(1)), int_or_none(crg.group(2)), int_or_none(crg.group(3))
 
 
+def read_stdin(what):
+    eof = 'Ctrl+Z' if compat_os_name == 'nt' else 'Ctrl+D'
+    write_string(f'Reading {what} from STDIN - EOF ({eof}) to end:\n')
+    return sys.stdin
+
+
 class Config:
     own_args = None
     parsed_args = None
@@ -5188,6 +5194,9 @@ class Config:
         self.parsed_args, self.filename = args, filename
 
         for location in opts.config_locations or []:
+            if location == '-':
+                self.append_config(shlex.split(read_stdin('options'), comments=True), label='stdin')
+                continue
             location = os.path.join(directory, expand_path(location))
             if os.path.isdir(location):
                 location = os.path.join(location, 'yt-dlp.conf')
-- 
cgit v1.2.3


From 8a82af3511b4379af0d239dbd01c672c17a2c46a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 27 May 2022 04:36:23 +0530
Subject: [cleanup] Misc fixes and cleanup

Closes #3780, Closes #3853, Closes #3850
---
 yt_dlp/utils.py | 61 ++++++++++++++++++++++++++++++---------------------------
 1 file changed, 32 insertions(+), 29 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 6701492f2..9da8bb293 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -619,9 +619,9 @@ def sanitize_open(filename, open_mode):
                     # Ref: https://github.com/yt-dlp/yt-dlp/issues/3124
                     raise LockingUnsupportedError()
                 stream = locked_file(filename, open_mode, block=False).__enter__()
-            except LockingUnsupportedError:
+            except OSError:
                 stream = open(filename, open_mode)
-            return (stream, filename)
+            return stream, filename
         except OSError as err:
             if attempt or err.errno in (errno.EACCES,):
                 raise
@@ -815,12 +815,9 @@ def escapeHTML(text):
 
 
 def process_communicate_or_kill(p, *args, **kwargs):
-    try:
-        return p.communicate(*args, **kwargs)
-    except BaseException:  # Including KeyboardInterrupt
-        p.kill()
-        p.wait()
-        raise
+    write_string('DeprecationWarning: yt_dlp.utils.process_communicate_or_kill is deprecated '
+                 'and may be removed in a future version. Use yt_dlp.utils.Popen.communicate_or_kill instead')
+    return Popen.communicate_or_kill(p, *args, **kwargs)
 
 
 class Popen(subprocess.Popen):
@@ -834,7 +831,12 @@ class Popen(subprocess.Popen):
         super().__init__(*args, **kwargs, startupinfo=self._startupinfo)
 
     def communicate_or_kill(self, *args, **kwargs):
-        return process_communicate_or_kill(self, *args, **kwargs)
+        try:
+            return self.communicate(*args, **kwargs)
+        except BaseException:  # Including KeyboardInterrupt
+            self.kill()
+            self.wait()
+            raise
 
 
 def get_subprocess_encoding():
@@ -921,22 +923,23 @@ def make_HTTPS_handler(params, **kwargs):
         context.options |= 4  # SSL_OP_LEGACY_SERVER_CONNECT
         # Allow use of weaker ciphers in Python 3.10+. See https://bugs.python.org/issue43998
         context.set_ciphers('DEFAULT')
+
     context.verify_mode = ssl.CERT_REQUIRED if opts_check_certificate else ssl.CERT_NONE
     if opts_check_certificate:
         if has_certifi and 'no-certifi' not in params.get('compat_opts', []):
             context.load_verify_locations(cafile=certifi.where())
-        else:
-            try:
-                context.load_default_certs()
-                # Work around the issue in load_default_certs when there are bad certificates. See:
-                # https://github.com/yt-dlp/yt-dlp/issues/1060,
-                # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
-            except ssl.SSLError:
-                # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
-                if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
-                    for storename in ('CA', 'ROOT'):
-                        _ssl_load_windows_store_certs(context, storename)
-                context.set_default_verify_paths()
+        try:
+            context.load_default_certs()
+        # Work around the issue in load_default_certs when there are bad certificates. See:
+        # https://github.com/yt-dlp/yt-dlp/issues/1060,
+        # https://bugs.python.org/issue35665, https://bugs.python.org/issue45312
+        except ssl.SSLError:
+            # enum_certificates is not present in mingw python. See https://github.com/yt-dlp/yt-dlp/issues/1151
+            if sys.platform == 'win32' and hasattr(ssl, 'enum_certificates'):
+                for storename in ('CA', 'ROOT'):
+                    _ssl_load_windows_store_certs(context, storename)
+            context.set_default_verify_paths()
+
     client_certfile = params.get('client_certificate')
     if client_certfile:
         try:
@@ -1885,11 +1888,11 @@ def platform_name():
 
 @functools.cache
 def get_windows_version():
-    ''' Get Windows version. None if it's not running on Windows '''
+    ''' Get Windows version. returns () if it's not running on Windows '''
     if compat_os_name == 'nt':
         return version_tuple(platform.win32_ver()[1])
     else:
-        return None
+        return ()
 
 
 def write_string(s, out=None, encoding=None):
@@ -1899,14 +1902,14 @@ def write_string(s, out=None, encoding=None):
     if compat_os_name == 'nt' and supports_terminal_sequences(out):
         s = re.sub(r'([\r\n]+)', r' \1', s)
 
-    enc = None
+    enc, buffer = None, out
     if 'b' in getattr(out, 'mode', ''):
         enc = encoding or preferredencoding()
     elif hasattr(out, 'buffer'):
-        out = out.buffer
+        buffer = out.buffer
         enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
 
-    out.write(s.encode(enc, 'ignore') if enc else s)
+    buffer.write(s.encode(enc, 'ignore') if enc else s)
     out.flush()
 
 
@@ -1925,7 +1928,7 @@ def intlist_to_bytes(xs):
     return compat_struct_pack('%dB' % len(xs), *xs)
 
 
-class LockingUnsupportedError(IOError):
+class LockingUnsupportedError(OSError):
     msg = 'File locking is not supported on this platform'
 
     def __init__(self):
@@ -5089,7 +5092,7 @@ WINDOWS_VT_MODE = False if compat_os_name == 'nt' else None
 @functools.cache
 def supports_terminal_sequences(stream):
     if compat_os_name == 'nt':
-        if not WINDOWS_VT_MODE or get_windows_version() < (10, 0, 10586):
+        if not WINDOWS_VT_MODE:
             return False
     elif not os.getenv('TERM'):
         return False
@@ -5100,7 +5103,7 @@ def supports_terminal_sequences(stream):
 
 
 def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.python.org/issue30075
-    if compat_os_name != 'nt':
+    if get_windows_version() < (10, 0, 10586):
         return
     global WINDOWS_VT_MODE
     startupinfo = subprocess.STARTUPINFO()
-- 
cgit v1.2.3


From 2c6dcb65fb612fc5bc5c61937bf438d3c473d8d0 Mon Sep 17 00:00:00 2001
From: coletdev <coletdjnz@protonmail.com>
Date: Sat, 28 May 2022 15:46:36 +1200
Subject: [utils] Send HTTP/1.1 ALPN extension (#3889)

Some servers may reject requests if not sent (e.g. fingerprinting)

Fixes #3878

Authored by: coletdjnz
---
 yt_dlp/utils.py | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 9da8bb293..b0300b724 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -948,6 +948,13 @@ def make_HTTPS_handler(params, **kwargs):
                 password=params.get('client_certificate_password'))
         except ssl.SSLError:
             raise YoutubeDLError('Unable to load client certificate')
+
+    # Some servers may reject requests if ALPN extension is not sent. See:
+    # https://github.com/python/cpython/issues/85140
+    # https://github.com/yt-dlp/yt-dlp/issues/3878
+    with contextlib.suppress(NotImplementedError):
+        context.set_alpn_protocols(['http/1.1'])
+
     return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
-- 
cgit v1.2.3


From 1890fc6389393ffaa05fa27bd47717f4d862404f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Jun 2022 21:29:03 +0530
Subject: [cleanup] Misc fixes

Cherry-picks from: #3498, #3947
Related: #3949, https://github.com/yt-dlp/yt-dlp/issues/1839#issuecomment-1140313836
Authored by: pukkandan, flashdagger, gamer191
---
 yt_dlp/utils.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index b0300b724..00721eb46 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1936,7 +1936,7 @@ def intlist_to_bytes(xs):
 
 
 class LockingUnsupportedError(OSError):
-    msg = 'File locking is not supported on this platform'
+    msg = 'File locking is not supported'
 
     def __init__(self):
         super().__init__(self.msg)
@@ -2061,8 +2061,11 @@ class locked_file:
             try:
                 self.f.truncate()
             except OSError as e:
-                if e.errno != 29:  # Illegal seek, expected when self.f is a FIFO
-                    raise e
+                if e.errno not in (
+                    errno.ESPIPE,  # Illegal seek - expected for FIFO
+                    errno.EINVAL,  # Invalid argument - expected for /dev/null
+                ):
+                    raise
         return self
 
     def unlock(self):
-- 
cgit v1.2.3


From b7c47b743871cdf3e0de75b17e4454d987384bf9 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Jun 2022 21:02:31 +0530
Subject: [extractor] Add `_search_json`

All fetching of JSON objects should eventually be done with this function
but only `youtube` is being refactored for now
---
 yt_dlp/utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 00721eb46..777b8b3ea 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -594,6 +594,19 @@ def clean_html(html):
     return html.strip()
 
 
+class LenientJSONDecoder(json.JSONDecoder):
+    def __init__(self, *args, transform_source=None, ignore_extra=False, **kwargs):
+        self.transform_source, self.ignore_extra = transform_source, ignore_extra
+        super().__init__(*args, **kwargs)
+
+    def decode(self, s):
+        if self.transform_source:
+            s = self.transform_source(s)
+        if self.ignore_extra:
+            return self.raw_decode(s.lstrip())[0]
+        return super().decode(s)
+
+
 def sanitize_open(filename, open_mode):
     """Try to open the given filename, and slightly tweak it if this fails.
 
-- 
cgit v1.2.3


From 5ec1b6b71689d2f0cbdcd2b6c4dd861fb2fcf911 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 7 Jun 2022 01:43:50 +0530
Subject: Add option `--download-sections` to download video partially

Closes #52, Closes #3932
---
 yt_dlp/utils.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 777b8b3ea..45af4ec61 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3495,6 +3495,23 @@ def match_filter_func(filters):
     return _match_func
 
 
+def download_range_func(chapters, ranges):
+    def inner(info_dict, ydl):
+        warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
+                   else 'Chapter information is unavailable')
+        for regex in chapters or []:
+            for i, chapter in enumerate(info_dict.get('chapters') or []):
+                if re.search(regex, chapter['title']):
+                    warning = None
+                    yield {**chapter, 'index': i}
+        if warning:
+            ydl.to_screen(f'[info] {info_dict["id"]}: {warning}')
+
+        yield from ({'start_time': start, 'end_time': end} for start, end in ranges or [])
+
+    return inner
+
+
 def parse_dfxp_time_expr(time_expr):
     if not time_expr:
         return
-- 
cgit v1.2.3


From 56ba69e4c991e81a449882258be08d0b6b98c648 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 11 Jun 2022 00:33:54 +0530
Subject: [cleanup] Misc fixes

Closes #4027
---
 yt_dlp/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 45af4ec61..137d29d0a 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3498,13 +3498,13 @@ def match_filter_func(filters):
 def download_range_func(chapters, ranges):
     def inner(info_dict, ydl):
         warning = ('There are no chapters matching the regex' if info_dict.get('chapters')
-                   else 'Chapter information is unavailable')
+                   else 'Cannot match chapters since chapter information is unavailable')
         for regex in chapters or []:
             for i, chapter in enumerate(info_dict.get('chapters') or []):
                 if re.search(regex, chapter['title']):
                     warning = None
                     yield {**chapter, 'index': i}
-        if warning:
+        if chapters and warning:
             ydl.to_screen(f'[info] {info_dict["id"]}: {warning}')
 
         yield from ({'start_time': start, 'end_time': end} for start, end in ranges or [])
@@ -4903,9 +4903,9 @@ def to_high_limit_path(path):
     return path
 
 
-def format_field(obj, field=None, template='%s', ignore=(None, ''), default='', func=None):
+def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=None):
     val = traverse_obj(obj, *variadic(field))
-    if val in ignore:
+    if (not val and val != 0) if ignore is NO_DEFAULT else val in ignore:
         return default
     return template % (func(val) if func else val)
 
-- 
cgit v1.2.3


From 64fa820ccf61a7aea6c2a48b1362b3a4ec270cad Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 25 May 2022 17:53:46 +0530
Subject: [cleanup] Misc fixes (see desc)

* [tvver] Fix bug in 6837633a4a614920b6e43ffc6b4b8590dca8c9d7 - Closes #4054
* [rumble] Fix tests - Closes #3976
* [make] Remove `cat` abuse - Closes #3989
* [make] Revert #3684 - Closes #3814
* [utils] Improve `get_elements_by_class` - Closes #3993
* [utils] Inherit `Namespace` from `types.SimpleNamespace`
* [utils] Use `re.fullmatch` for matching filters
* [jsinterp] Handle quotes in `_separate`
* [make_readme] Allow overshooting last line

Authored by: pukkandan, kwconder, MrRawes, Lesmiscore
---
 yt_dlp/utils.py | 33 ++++++++++++---------------------
 1 file changed, 12 insertions(+), 21 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 137d29d0a..e6e6d2759 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -34,6 +34,7 @@ import sys
 import tempfile
 import time
 import traceback
+import types
 import urllib.parse
 import xml.etree.ElementTree
 import zlib
@@ -397,14 +398,14 @@ def get_element_html_by_attribute(attribute, value, html, **kargs):
 def get_elements_by_class(class_name, html, **kargs):
     """Return the content of all tags with the specified class in the passed HTML document as a list"""
     return get_elements_by_attribute(
-        'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
+        'class', r'[^\'"]*(?<=[\'"\s])%s(?=[\'"\s])[^\'"]*' % re.escape(class_name),
         html, escape_value=False)
 
 
 def get_elements_html_by_class(class_name, html):
     """Return the html of all tags with the specified class in the passed HTML document as a list"""
     return get_elements_html_by_attribute(
-        'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
+        'class', r'[^\'"]*(?<=[\'"\s])%s(?=[\'"\s])[^\'"]*' % re.escape(class_name),
         html, escape_value=False)
 
 
@@ -3404,16 +3405,15 @@ def _match_one(filter_part, dct, incomplete):
     else:
         is_incomplete = lambda k: k in incomplete
 
-    operator_rex = re.compile(r'''(?x)\s*
+    operator_rex = re.compile(r'''(?x)
         (?P<key>[a-z_]+)
         \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
         (?:
             (?P<quote>["\'])(?P<quotedstrval>.+?)(?P=quote)|
             (?P<strval>.+?)
         )
-        \s*$
         ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
-    m = operator_rex.search(filter_part)
+    m = operator_rex.fullmatch(filter_part.strip())
     if m:
         m = m.groupdict()
         unnegated_op = COMPARISON_OPERATORS[m['op']]
@@ -3449,11 +3449,10 @@ def _match_one(filter_part, dct, incomplete):
         '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
         '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
     }
-    operator_rex = re.compile(r'''(?x)\s*
+    operator_rex = re.compile(r'''(?x)
         (?P<op>%s)\s*(?P<key>[a-z_]+)
-        \s*$
         ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
-    m = operator_rex.search(filter_part)
+    m = operator_rex.fullmatch(filter_part.strip())
     if m:
         op = UNARY_OPERATORS[m.group('op')]
         actual_value = dct.get(m.group('key'))
@@ -5395,23 +5394,15 @@ class classproperty:
         return self.func(cls)
 
 
-class Namespace:
+class Namespace(types.SimpleNamespace):
     """Immutable namespace"""
 
-    def __init__(self, **kwargs):
-        self._dict = kwargs
-
-    def __getattr__(self, attr):
-        return self._dict[attr]
-
-    def __contains__(self, item):
-        return item in self._dict.values()
-
     def __iter__(self):
-        return iter(self._dict.items())
+        return iter(self.__dict__.values())
 
-    def __repr__(self):
-        return f'{type(self).__name__}({", ".join(f"{k}={v}" for k, v in self)})'
+    @property
+    def items_(self):
+        return self.__dict__.items()
 
 
 # Deprecated
-- 
cgit v1.2.3


From 2cb19820430aa8f7fe8cef11203d9f98388ef8ab Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 13 Jun 2022 17:27:31 +0530
Subject: [utils] `locked_file`: Fix for PyPy on Windows

---
 yt_dlp/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index e6e6d2759..11ef7744c 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2003,7 +2003,8 @@ if sys.platform == 'win32':
         if not LockFileEx(msvcrt.get_osfhandle(f.fileno()),
                           (0x2 if exclusive else 0x0) | (0x0 if block else 0x1),
                           0, whole_low, whole_high, f._lock_file_overlapped_p):
-            raise BlockingIOError('Locking file failed: %r' % ctypes.FormatError())
+            # NB: No argument form of "ctypes.FormatError" does not work on PyPy
+            raise BlockingIOError(f'Locking file failed: {ctypes.FormatError(ctypes.GetLastError())!r}')
 
     def _unlock_file(f):
         assert f._lock_file_overlapped_p
-- 
cgit v1.2.3


From f0c9fb96827ff798a48626e7e5d32a9c5de7b97e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 16 Jun 2022 02:25:43 +0530
Subject: [utils] `Popen`: Refactor to use contextmanager

Fixes https://github.com/yt-dlp/yt-dlp/issues/3531#issuecomment-1156223597
---
 yt_dlp/utils.py | 42 +++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 11ef7744c..be7cbf9fd 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -841,17 +841,31 @@ class Popen(subprocess.Popen):
     else:
         _startupinfo = None
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, text=False, **kwargs):
+        if text is True:
+            kwargs['universal_newlines'] = True  # For 3.6 compatibility
+            kwargs.setdefault('encoding', 'utf-8')
+            kwargs.setdefault('errors', 'replace')
         super().__init__(*args, **kwargs, startupinfo=self._startupinfo)
 
     def communicate_or_kill(self, *args, **kwargs):
         try:
             return self.communicate(*args, **kwargs)
         except BaseException:  # Including KeyboardInterrupt
-            self.kill()
-            self.wait()
+            self.kill(timeout=None)
             raise
 
+    def kill(self, *, timeout=0):
+        super().kill()
+        if timeout != 0:
+            self.wait(timeout=timeout)
+
+    @classmethod
+    def run(cls, *args, **kwargs):
+        with cls(*args, **kwargs) as proc:
+            stdout, stderr = proc.communicate_or_kill()
+            return stdout or '', stderr or '', proc.returncode
+
 
 def get_subprocess_encoding():
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
@@ -2556,7 +2570,7 @@ def check_executable(exe, args=[]):
     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
     args can be a list of arguments for a short output (like -version) """
     try:
-        Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate_or_kill()
+        Popen.run([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     except OSError:
         return False
     return exe
@@ -2569,14 +2583,11 @@ def _get_exe_version_output(exe, args, *, to_screen=None):
         # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
         # SIGTTOU if yt-dlp is run in the background.
         # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
-        out, _ = Popen(
-            [encodeArgument(exe)] + args, stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate_or_kill()
+        stdout, _, _ = Popen.run([encodeArgument(exe)] + args, text=True,
+                                 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     except OSError:
         return False
-    if isinstance(out, bytes):  # Python 2.x
-        out = out.decode('ascii', 'ignore')
-    return out
+    return stdout
 
 
 def detect_exe_version(output, version_re=None, unrecognized='present'):
@@ -4796,14 +4807,13 @@ def write_xattr(path, key, value):
 
     value = value.decode()
     try:
-        p = Popen(
+        _, stderr, returncode = Popen.run(
             [exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
     except OSError as e:
         raise XAttrMetadataError(e.errno, e.strerror)
-    stderr = p.communicate_or_kill()[1].decode('utf-8', 'replace')
-    if p.returncode:
-        raise XAttrMetadataError(p.returncode, stderr)
+    if returncode:
+        raise XAttrMetadataError(returncode, stderr)
 
 
 def random_birthday(year_field, month_field, day_field):
@@ -5146,10 +5156,8 @@ def windows_enable_vt_mode():  # TODO: Do this the proper way https://bugs.pytho
     if get_windows_version() < (10, 0, 10586):
         return
     global WINDOWS_VT_MODE
-    startupinfo = subprocess.STARTUPINFO()
-    startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
     try:
-        subprocess.Popen('', shell=True, startupinfo=startupinfo).wait()
+        Popen.run('', shell=True)
     except Exception:
         return
 
-- 
cgit v1.2.3


From 7e88d7d78f452ea69f06bbdf23f82e9ad7c3de5e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Jun 2022 10:18:21 +0530
Subject: Add slicing notation to `--playlist-items`

* Adds support for negative indices and step
* Add `-I` as alias for `--playlist-index`
* Deprecates `--playlist-start`, `--playlist-end`, `--playlist-reverse`, `--no-playlist-reverse`

Closes #2951, Closes #2853
---
 yt_dlp/utils.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 152 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index be7cbf9fd..f21d70672 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2609,6 +2609,16 @@ def get_exe_version(exe, args=['--version'],
     return detect_exe_version(out, version_re, unrecognized) if out else False
 
 
+def frange(start=0, stop=None, step=1):
+    """Float range"""
+    if stop is None:
+        start, stop = 0, start
+    sign = [-1, 1][step > 0] if step else 0
+    while sign * start < sign * stop:
+        yield start
+        start += step
+
+
 class LazyList(collections.abc.Sequence):
     """Lazy immutable list from an iterable
     Note that slices of a LazyList are lists and not LazyList"""
@@ -2805,6 +2815,148 @@ class InAdvancePagedList(PagedList):
             yield from page_results
 
 
+class PlaylistEntries:
+    MissingEntry = object()
+    is_exhausted = False
+
+    def __init__(self, ydl, info_dict):
+        self.ydl, self.info_dict = ydl, info_dict
+
+    PLAYLIST_ITEMS_RE = re.compile(r'''(?x)
+        (?P<start>[+-]?\d+)?
+        (?P<range>[:-]
+            (?P<end>[+-]?\d+|inf(?:inite)?)?
+            (?::(?P<step>[+-]?\d+))?
+        )?''')
+
+    @classmethod
+    def parse_playlist_items(cls, string):
+        for segment in string.split(','):
+            if not segment:
+                raise ValueError('There is two or more consecutive commas')
+            mobj = cls.PLAYLIST_ITEMS_RE.fullmatch(segment)
+            if not mobj:
+                raise ValueError(f'{segment!r} is not a valid specification')
+            start, end, step, has_range = mobj.group('start', 'end', 'step', 'range')
+            if int_or_none(step) == 0:
+                raise ValueError(f'Step in {segment!r} cannot be zero')
+            yield slice(int_or_none(start), float_or_none(end), int_or_none(step)) if has_range else int(start)
+
+    def get_requested_items(self):
+        playlist_items = self.ydl.params.get('playlist_items')
+        playlist_start = self.ydl.params.get('playliststart', 1)
+        playlist_end = self.ydl.params.get('playlistend')
+        # For backwards compatibility, interpret -1 as whole list
+        if playlist_end in (-1, None):
+            playlist_end = ''
+        if not playlist_items:
+            playlist_items = f'{playlist_start}:{playlist_end}'
+        elif playlist_start != 1 or playlist_end:
+            self.ydl.report_warning('Ignoring playliststart and playlistend because playlistitems was given', only_once=True)
+
+        for index in self.parse_playlist_items(playlist_items):
+            for i, entry in self[index]:
+                yield i, entry
+                try:
+                    # TODO: Add auto-generated fields
+                    self.ydl._match_entry(entry, incomplete=True, silent=True)
+                except (ExistingVideoReached, RejectedVideoReached):
+                    return
+
+    @property
+    def full_count(self):
+        if self.info_dict.get('playlist_count'):
+            return self.info_dict['playlist_count']
+        elif self.is_exhausted and not self.is_incomplete:
+            return len(self)
+        elif isinstance(self._entries, InAdvancePagedList):
+            if self._entries._pagesize == 1:
+                return self._entries._pagecount
+
+    @functools.cached_property
+    def _entries(self):
+        entries = self.info_dict.get('entries')
+        if entries is None:
+            raise EntryNotInPlaylist('There are no entries')
+        elif isinstance(entries, list):
+            self.is_exhausted = True
+
+        indices = self.info_dict.get('requested_entries')
+        self.is_incomplete = bool(indices)
+        if self.is_incomplete:
+            assert self.is_exhausted
+            ret = [self.MissingEntry] * max(indices)
+            for i, entry in zip(indices, entries):
+                ret[i - 1] = entry
+            return ret
+
+        if isinstance(entries, (list, PagedList, LazyList)):
+            return entries
+        return LazyList(entries)
+
+    @functools.cached_property
+    def _getter(self):
+        if isinstance(self._entries, list):
+            def get_entry(i):
+                try:
+                    entry = self._entries[i]
+                except IndexError:
+                    entry = self.MissingEntry
+                    if not self.is_incomplete:
+                        raise self.IndexError()
+                if entry is self.MissingEntry:
+                    raise EntryNotInPlaylist(f'Entry {i} cannot be found')
+                return entry
+        else:
+            def get_entry(i):
+                try:
+                    return type(self.ydl)._handle_extraction_exceptions(lambda _, i: self._entries[i])(self.ydl, i)
+                except (LazyList.IndexError, PagedList.IndexError):
+                    raise self.IndexError()
+        return get_entry
+
+    def __getitem__(self, idx):
+        if isinstance(idx, int):
+            idx = slice(idx, idx)
+
+        # NB: PlaylistEntries[1:10] => (0, 1, ... 9)
+        step = 1 if idx.step is None else idx.step
+        if idx.start is None:
+            start = 0 if step > 0 else len(self) - 1
+        else:
+            start = idx.start - 1 if idx.start >= 0 else len(self) + idx.start
+
+        # NB: Do not call len(self) when idx == [:]
+        if idx.stop is None:
+            stop = 0 if step < 0 else float('inf')
+        else:
+            stop = idx.stop - 1 if idx.stop >= 0 else len(self) + idx.stop
+        stop += [-1, 1][step > 0]
+
+        for i in frange(start, stop, step):
+            if i < 0:
+                continue
+            try:
+                try:
+                    entry = self._getter(i)
+                except self.IndexError:
+                    self.is_exhausted = True
+                    if step > 0:
+                        break
+                    continue
+            except IndexError:
+                if self.is_exhausted:
+                    break
+                raise
+            yield i + 1, entry
+
+    def __len__(self):
+        return len(tuple(self[:]))
+
+    class IndexError(IndexError):
+        pass
+
+
 def uppercase_escape(s):
     unicode_escape = codecs.getdecoder('unicode_escape')
     return re.sub(
-- 
cgit v1.2.3


From 7e9a61258543f64113e779f2f82fe7a29827489d Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 17 Jun 2022 13:35:04 +0530
Subject: Add option `--lazy-playlist` to process entries as they are received

---
 yt_dlp/utils.py | 81 ++++++++++++++++++++++++++-------------------------------
 1 file changed, 37 insertions(+), 44 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index f21d70672..8dda5e931 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -770,13 +770,16 @@ def expand_path(s):
     return os.path.expandvars(compat_expanduser(s))
 
 
-def orderedSet(iterable):
-    """ Remove all duplicates from the input iterable """
-    res = []
-    for el in iterable:
-        if el not in res:
-            res.append(el)
-    return res
+def orderedSet(iterable, *, lazy=False):
+    """Remove all duplicates from the input iterable"""
+    def _iter():
+        seen = []  # Do not use set since the items can be unhashable
+        for x in iterable:
+            if x not in seen:
+                seen.append(x)
+                yield x
+
+    return _iter() if lazy else list(_iter())
 
 
 def _htmlentity_transform(entity_with_semicolon):
@@ -2820,7 +2823,26 @@ class PlaylistEntries:
     is_exhausted = False
 
     def __init__(self, ydl, info_dict):
-        self.ydl, self.info_dict = ydl, info_dict
+        self.ydl = ydl
+
+        # _entries must be assigned now since infodict can change during iteration
+        entries = info_dict.get('entries')
+        if entries is None:
+            raise EntryNotInPlaylist('There are no entries')
+        elif isinstance(entries, list):
+            self.is_exhausted = True
+
+        requested_entries = info_dict.get('requested_entries')
+        self.is_incomplete = bool(requested_entries)
+        if self.is_incomplete:
+            assert self.is_exhausted
+            self._entries = [self.MissingEntry] * max(requested_entries)
+            for i, entry in zip(requested_entries, entries):
+                self._entries[i - 1] = entry
+        elif isinstance(entries, (list, PagedList, LazyList)):
+            self._entries = entries
+        else:
+            self._entries = LazyList(entries)
 
     PLAYLIST_ITEMS_RE = re.compile(r'''(?x)
         (?P<start>[+-]?\d+)?
@@ -2863,37 +2885,13 @@ class PlaylistEntries:
                 except (ExistingVideoReached, RejectedVideoReached):
                     return
 
-    @property
-    def full_count(self):
-        if self.info_dict.get('playlist_count'):
-            return self.info_dict['playlist_count']
-        elif self.is_exhausted and not self.is_incomplete:
+    def get_full_count(self):
+        if self.is_exhausted and not self.is_incomplete:
             return len(self)
         elif isinstance(self._entries, InAdvancePagedList):
             if self._entries._pagesize == 1:
                 return self._entries._pagecount
 
-    @functools.cached_property
-    def _entries(self):
-        entries = self.info_dict.get('entries')
-        if entries is None:
-            raise EntryNotInPlaylist('There are no entries')
-        elif isinstance(entries, list):
-            self.is_exhausted = True
-
-        indices = self.info_dict.get('requested_entries')
-        self.is_incomplete = bool(indices)
-        if self.is_incomplete:
-            assert self.is_exhausted
-            ret = [self.MissingEntry] * max(indices)
-            for i, entry in zip(indices, entries):
-                ret[i - 1] = entry
-            return ret
-
-        if isinstance(entries, (list, PagedList, LazyList)):
-            return entries
-        return LazyList(entries)
-
     @functools.cached_property
     def _getter(self):
         if isinstance(self._entries, list):
@@ -2937,17 +2935,12 @@ class PlaylistEntries:
             if i < 0:
                 continue
             try:
-                try:
-                    entry = self._getter(i)
-                except self.IndexError:
-                    self.is_exhausted = True
-                    if step > 0:
-                        break
-                    continue
-            except IndexError:
-                if self.is_exhausted:
+                entry = self._getter(i)
+            except self.IndexError:
+                self.is_exhausted = True
+                if step > 0:
                     break
-                raise
+                continue
             yield i + 1, entry
 
     def __len__(self):
-- 
cgit v1.2.3


From e121e3cee731426f620e17939141018d09661fa2 Mon Sep 17 00:00:00 2001
From: christoph-heinrich <christoph-heinrich@users.noreply.github.com>
Date: Sat, 18 Jun 2022 03:57:22 +0200
Subject: [cleanup] Minor fixes (#4096)

Authored by: christoph-heinrich
---
 yt_dlp/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 8dda5e931..4a519e4e0 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -4954,7 +4954,7 @@ def write_xattr(path, key, value):
     try:
         _, stderr, returncode = Popen.run(
             [exe, '-w', key, value, path] if exe == 'xattr' else [exe, '-n', key, '-v', value, path],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+            text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
     except OSError as e:
         raise XAttrMetadataError(e.errno, e.strerror)
     if returncode:
-- 
cgit v1.2.3


From a70635b8a1bcf42bf587fe3cd7503f1d092009ce Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 18 Jun 2022 07:30:12 +0530
Subject: [cleanup, utils] Don't use kwargs for `format_field`

---
 yt_dlp/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 4a519e4e0..ea5bb3459 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1038,10 +1038,10 @@ class ExtractorError(YoutubeDLError):
         self.exc_info = sys.exc_info()  # preserve original exception
 
         super().__init__(''.join((
-            format_field(ie, template='[%s] '),
-            format_field(video_id, template='%s: '),
+            format_field(ie, None, '[%s] '),
+            format_field(video_id, None, '%s: '),
             msg,
-            format_field(cause, template=' (caused by %r)'),
+            format_field(cause, None, ' (caused by %r)'),
             '' if expected else bug_reports_message())))
 
     def format_traceback(self):
-- 
cgit v1.2.3


From 44a6fcff397e98b4aa7e3bb1da7425b3cca05a71 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 18 Jun 2022 09:17:45 +0530
Subject: Improve error handling of bad config files

Related: #824
---
 yt_dlp/utils.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index ea5bb3459..72223d771 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -5420,6 +5420,8 @@ class Config:
             # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
             contents = optionf.read()
             res = shlex.split(contents, comments=True)
+        except Exception as err:
+            raise ValueError(f'Unable to parse "{filename}": {err}')
         finally:
             optionf.close()
         return res
-- 
cgit v1.2.3


From 8072ef2bbd1721e4c79156b422e4fccc1e062853 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 20 Jun 2022 03:03:19 +0530
Subject: [extractor/BiliIntl] Fix metadata extraction

Closes #4116
---
 yt_dlp/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 72223d771..7614839fb 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3216,7 +3216,11 @@ def js_to_json(code, vars={}):
 
         return '"%s"' % v
 
+    def create_map(mobj):
+        return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=vars))))
+
     code = re.sub(r'new Date\((".+")\)', r'\g<1>', code)
+    code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
 
     return re.sub(r'''(?sx)
         "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
-- 
cgit v1.2.3


From 8aa0e7cd96a1e2f315d49744793ae07f6543ce4c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 20 Jun 2022 10:48:29 +0530
Subject: [docs] Improvements

---
 yt_dlp/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 7614839fb..6abdca788 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3243,7 +3243,7 @@ def qualities(quality_ids):
     return q
 
 
-POSTPROCESS_WHEN = ('pre_process', 'after_filter', 'before_dl', 'after_move', 'post_process', 'after_video', 'playlist')
+POSTPROCESS_WHEN = ('pre_process', 'after_filter', 'before_dl', 'post_process', 'after_move', 'after_video', 'playlist')
 
 
 DEFAULT_OUTTMPL = {
-- 
cgit v1.2.3


From 7b2c3f47c6b586a208655fcfc716bba3f8619d1e Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 20 Jun 2022 11:44:55 +0530
Subject: [cleanup] Misc

---
 yt_dlp/utils.py | 66 ++++++++++++++++++++++++++++++++-------------------------
 1 file changed, 37 insertions(+), 29 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 6abdca788..b9c579cb6 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -146,6 +146,7 @@ USER_AGENTS = {
 
 
 NO_DEFAULT = object()
+IDENTITY = lambda x: x
 
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
@@ -4744,22 +4745,42 @@ def pkcs1pad(data, length):
     return [0, 2] + pseudo_random + [0] + data
 
 
-def encode_base_n(num, n, table=None):
-    FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-    if not table:
-        table = FULL_TABLE[:n]
+def _base_n_table(n, table):
+    if not table and not n:
+        raise ValueError('Either table or n must be specified')
+    elif not table:
+        table = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'[:n]
+    elif not n or n == len(table):
+        return table
+    raise ValueError(f'base {n} exceeds table length {len(table)}')
 
-    if n > len(table):
-        raise ValueError('base %d exceeds table length %d' % (n, len(table)))
 
-    if num == 0:
+def encode_base_n(num, n=None, table=None):
+    """Convert given int to a base-n string"""
+    table = _base_n_table(n)
+    if not num:
         return table[0]
 
-    ret = ''
+    result, base = '', len(table)
     while num:
-        ret = table[num % n] + ret
-        num = num // n
-    return ret
+        result = table[num % base] + result
+        num = num // result
+    return result
+
+
+def decode_base_n(string, n=None, table=None):
+    """Convert given base-n string to int"""
+    table = {char: index for index, char in enumerate(_base_n_table(n, table))}
+    result, base = 0, len(table)
+    for char in string:
+        result = result * base + table[char]
+    return result
+
+
+def decode_base(value, digits):
+    write_string('DeprecationWarning: yt_dlp.utils.decode_base is deprecated '
+                 'and may be removed in a future version. Use yt_dlp.decode_base_n instead')
+    return decode_base_n(value, table=digits)
 
 
 def decode_packed_codes(code):
@@ -5062,11 +5083,11 @@ def to_high_limit_path(path):
     return path
 
 
-def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=None):
+def format_field(obj, field=None, template='%s', ignore=NO_DEFAULT, default='', func=IDENTITY):
     val = traverse_obj(obj, *variadic(field))
-    if (not val and val != 0) if ignore is NO_DEFAULT else val in ignore:
+    if (not val and val != 0) if ignore is NO_DEFAULT else val in variadic(ignore):
         return default
-    return template % (func(val) if func else val)
+    return template % func(val)
 
 
 def clean_podcast_url(url):
@@ -5207,10 +5228,8 @@ def traverse_obj(
 
     if isinstance(expected_type, type):
         type_test = lambda val: val if isinstance(val, expected_type) else None
-    elif expected_type is not None:
-        type_test = expected_type
     else:
-        type_test = lambda val: val
+        type_test = expected_type or IDENTITY
 
     for path in path_list:
         depth = 0
@@ -5243,17 +5262,6 @@ def variadic(x, allowed_types=(str, bytes, dict)):
     return x if isinstance(x, collections.abc.Iterable) and not isinstance(x, allowed_types) else (x,)
 
 
-def decode_base(value, digits):
-    # This will convert given base-x string to scalar (long or int)
-    table = {char: index for index, char in enumerate(digits)}
-    result = 0
-    base = len(digits)
-    for chr in value:
-        result *= base
-        result += table[chr]
-    return result
-
-
 def time_seconds(**kwargs):
     t = datetime.datetime.now(datetime.timezone(datetime.timedelta(**kwargs)))
     return t.timestamp()
@@ -5327,7 +5335,7 @@ def number_of_digits(number):
 
 def join_nonempty(*values, delim='-', from_dict=None):
     if from_dict is not None:
-        values = map(from_dict.get, values)
+        values = (traverse_obj(from_dict, variadic(v)) for v in values)
     return delim.join(map(str, filter(None, values)))
 
 
-- 
cgit v1.2.3


From 612f2be5d3924540158dfbe5f25d841f04cff8c6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 20 Jun 2022 11:55:54 +0530
Subject: Bugfix for 7b2c3f47c6b586a208655fcfc716bba3f8619d1e

---
 yt_dlp/utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index b9c579cb6..9c16d6601 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -4748,23 +4748,23 @@ def pkcs1pad(data, length):
 def _base_n_table(n, table):
     if not table and not n:
         raise ValueError('Either table or n must be specified')
-    elif not table:
-        table = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'[:n]
-    elif not n or n == len(table):
-        return table
-    raise ValueError(f'base {n} exceeds table length {len(table)}')
+    table = (table or '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ')[:n]
+
+    if n != len(table):
+        raise ValueError(f'base {n} exceeds table length {len(table)}')
+    return table
 
 
 def encode_base_n(num, n=None, table=None):
     """Convert given int to a base-n string"""
-    table = _base_n_table(n)
+    table = _base_n_table(n, table)
     if not num:
         return table[0]
 
     result, base = '', len(table)
     while num:
         result = table[num % base] + result
-        num = num // result
+        num = num // base
     return result
 
 
-- 
cgit v1.2.3


From 5df14442552038d7344162b21f97dd510fe2ffd6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Mon, 20 Jun 2022 12:30:02 +0530
Subject: [utils] `ExtractorError`: Fix `exc_info`

---
 yt_dlp/utils.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 9c16d6601..10bcd5f4e 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1037,6 +1037,8 @@ class ExtractorError(YoutubeDLError):
         self.video_id = video_id
         self.ie = ie
         self.exc_info = sys.exc_info()  # preserve original exception
+        if isinstance(self.exc_info[1], ExtractorError):
+            self.exc_info = self.exc_info[1].exc_info
 
         super().__init__(''.join((
             format_field(ie, None, '[%s] '),
-- 
cgit v1.2.3


From 57e0f077a635ee30f37ebea71ddb70723831ecd8 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 21 Jun 2022 17:02:56 +0530
Subject: [update] Expose more functionality to API

---
 yt_dlp/utils.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 10bcd5f4e..dc6894d83 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -991,9 +991,10 @@ def make_HTTPS_handler(params, **kwargs):
 
 
 def bug_reports_message(before=';'):
-    msg = ('please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , '
-           'filling out the appropriate issue template. '
-           'Confirm you are on the latest version using  yt-dlp -U')
+    from .update import REPOSITORY
+
+    msg = (f'please report this issue on  https://github.com/{REPOSITORY}/issues?q= , '
+           'filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U')
 
     before = before.rstrip()
     if not before or before.endswith(('.', '!', '?')):
-- 
cgit v1.2.3


From 1ac4fd80c87d4e566ae680076e788a63d187199b Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 22 Jun 2022 08:39:14 +0530
Subject: Fix playlist error handling

Bug in 7e88d7d78f452ea69f06bbdf23f82e9ad7c3de5e
---
 yt_dlp/utils.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index dc6894d83..4dfdbd58b 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -2883,6 +2883,8 @@ class PlaylistEntries:
         for index in self.parse_playlist_items(playlist_items):
             for i, entry in self[index]:
                 yield i, entry
+                if not entry:
+                    continue
                 try:
                     # TODO: Add auto-generated fields
                     self.ydl._match_entry(entry, incomplete=True, silent=True)
-- 
cgit v1.2.3


From 379a4f161d4ad3e40932dcf5aca6e6fb9715ab28 Mon Sep 17 00:00:00 2001
From: coletdev <coletdjnz@protonmail.com>
Date: Fri, 24 Jun 2022 03:29:28 +0000
Subject: [utils] Fix inconsistent default handling between HTTP and HTTPS
 requests (#4158)

Default headers such as `Content-Type` were only being added for HTTPS requests among other handling.

Fixes bug in https://github.com/ytdl-org/youtube-dl/commit/be4a824d74add1a3b78b8244dff12f4f078f168a

Authored-by: coletdjnz
---
 yt_dlp/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 4dfdbd58b..3fc4961dd 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1343,7 +1343,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
         req.headers = handle_youtubedl_headers(req.headers)
 
-        return req
+        return super().do_request_(req)
 
     def http_response(self, req, resp):
         old_resp = resp
-- 
cgit v1.2.3


From ac668111128b5f124b4271b3aa4c35f6e71a4749 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 13:40:17 +0530
Subject: [compat] Remove more functions

Removing any more will require changes to a large number of extractors
---
 yt_dlp/utils.py | 124 +++++++++++++++++++++++++++-----------------------------
 1 file changed, 59 insertions(+), 65 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 3fc4961dd..6b02eb450 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -14,6 +14,8 @@ import errno
 import gzip
 import hashlib
 import hmac
+import html.entities
+import html.parser
 import importlib.util
 import io
 import itertools
@@ -29,6 +31,7 @@ import re
 import shlex
 import socket
 import ssl
+import struct
 import subprocess
 import sys
 import tempfile
@@ -36,35 +39,27 @@ import time
 import traceback
 import types
 import urllib.parse
+import urllib.request
 import xml.etree.ElementTree
 import zlib
+import http.client
+import http.cookiejar
 
 from .compat import asyncio, functools  # isort: split
 from .compat import (
-    compat_chr,
-    compat_cookiejar,
     compat_etree_fromstring,
     compat_expanduser,
-    compat_html_entities,
-    compat_html_entities_html5,
     compat_HTMLParseError,
-    compat_HTMLParser,
-    compat_http_client,
     compat_HTTPError,
     compat_os_name,
     compat_parse_qs,
     compat_shlex_quote,
     compat_str,
-    compat_struct_pack,
-    compat_struct_unpack,
-    compat_urllib_error,
-    compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
-    compat_urllib_request,
     compat_urlparse,
 )
-from .dependencies import brotli, certifi, websockets
+from .dependencies import brotli, certifi, websockets, xattr
 from .socks import ProxyType, sockssocket
 
 
@@ -445,7 +440,7 @@ def get_elements_text_and_html_by_attribute(attribute, value, html, escape_value
         )
 
 
-class HTMLBreakOnClosingTagParser(compat_HTMLParser):
+class HTMLBreakOnClosingTagParser(html.parser.HTMLParser):
     """
     HTML parser which raises HTMLBreakOnClosingTagException upon reaching the
     closing tag for the first opening tag it has encountered, and can be used
@@ -457,7 +452,7 @@ class HTMLBreakOnClosingTagParser(compat_HTMLParser):
 
     def __init__(self):
         self.tagstack = collections.deque()
-        compat_HTMLParser.__init__(self)
+        html.parser.HTMLParser.__init__(self)
 
     def __enter__(self):
         return self
@@ -522,22 +517,22 @@ def get_element_text_and_html_by_tag(tag, html):
         raise compat_HTMLParseError('unexpected end of html')
 
 
-class HTMLAttributeParser(compat_HTMLParser):
+class HTMLAttributeParser(html.parser.HTMLParser):
     """Trivial HTML parser to gather the attributes for a single element"""
 
     def __init__(self):
         self.attrs = {}
-        compat_HTMLParser.__init__(self)
+        html.parser.HTMLParser.__init__(self)
 
     def handle_starttag(self, tag, attrs):
         self.attrs = dict(attrs)
 
 
-class HTMLListAttrsParser(compat_HTMLParser):
+class HTMLListAttrsParser(html.parser.HTMLParser):
     """HTML parser to gather the attributes for the elements of a list"""
 
     def __init__(self):
-        compat_HTMLParser.__init__(self)
+        html.parser.HTMLParser.__init__(self)
         self.items = []
         self._level = 0
 
@@ -763,7 +758,7 @@ def sanitized_Request(url, *args, **kwargs):
     if auth_header is not None:
         headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
         headers['Authorization'] = auth_header
-    return compat_urllib_request.Request(url, *args, **kwargs)
+    return urllib.request.Request(url, *args, **kwargs)
 
 
 def expand_path(s):
@@ -788,13 +783,13 @@ def _htmlentity_transform(entity_with_semicolon):
     entity = entity_with_semicolon[:-1]
 
     # Known non-numeric HTML entity
-    if entity in compat_html_entities.name2codepoint:
-        return compat_chr(compat_html_entities.name2codepoint[entity])
+    if entity in html.entities.name2codepoint:
+        return chr(html.entities.name2codepoint[entity])
 
     # TODO: HTML5 allows entities without a semicolon. For example,
     # '&Eacuteric' should be decoded as 'Éric'.
-    if entity_with_semicolon in compat_html_entities_html5:
-        return compat_html_entities_html5[entity_with_semicolon]
+    if entity_with_semicolon in html.entities.html5:
+        return html.entities.html5[entity_with_semicolon]
 
     mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
     if mobj is not None:
@@ -806,7 +801,7 @@ def _htmlentity_transform(entity_with_semicolon):
             base = 10
         # See https://github.com/ytdl-org/youtube-dl/issues/7518
         with contextlib.suppress(ValueError):
-            return compat_chr(int(numstr, base))
+            return chr(int(numstr, base))
 
     # Unknown entity in name, return its literal representation
     return '&%s;' % entity
@@ -1015,7 +1010,7 @@ class YoutubeDLError(Exception):
         super().__init__(self.msg)
 
 
-network_exceptions = [compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error]
+network_exceptions = [urllib.error.URLError, http.client.HTTPException, socket.error]
 if hasattr(ssl, 'CertificateError'):
     network_exceptions.append(ssl.CertificateError)
 network_exceptions = tuple(network_exceptions)
@@ -1267,7 +1262,7 @@ def handle_youtubedl_headers(headers):
     return filtered_headers
 
 
-class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
+class YoutubeDLHandler(urllib.request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
@@ -1286,11 +1281,11 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """
 
     def __init__(self, params, *args, **kwargs):
-        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
+        urllib.request.HTTPHandler.__init__(self, *args, **kwargs)
         self._params = params
 
     def http_open(self, req):
-        conn_class = compat_http_client.HTTPConnection
+        conn_class = http.client.HTTPConnection
 
         socks_proxy = req.headers.get('Ytdl-socks-proxy')
         if socks_proxy:
@@ -1365,18 +1360,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                     break
                 else:
                     raise original_ioerror
-            resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
+            resp = urllib.request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # deflate
         if resp.headers.get('Content-encoding', '') == 'deflate':
             gz = io.BytesIO(self.deflate(resp.read()))
-            resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp = urllib.request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
         # brotli
         if resp.headers.get('Content-encoding', '') == 'br':
-            resp = compat_urllib_request.addinfourl(
+            resp = urllib.request.addinfourl(
                 io.BytesIO(self.brotli(resp.read())), old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
             del resp.headers['Content-encoding']
@@ -1399,7 +1394,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
 def make_socks_conn_class(base_class, socks_proxy):
     assert issubclass(base_class, (
-        compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
+        http.client.HTTPConnection, http.client.HTTPSConnection))
 
     url_components = compat_urlparse.urlparse(socks_proxy)
     if url_components.scheme.lower() == 'socks5':
@@ -1412,7 +1407,7 @@ def make_socks_conn_class(base_class, socks_proxy):
     def unquote_if_non_empty(s):
         if not s:
             return s
-        return compat_urllib_parse_unquote_plus(s)
+        return urllib.parse.unquote_plus(s)
 
     proxy_args = (
         socks_type,
@@ -1430,7 +1425,7 @@ def make_socks_conn_class(base_class, socks_proxy):
                 self.sock.settimeout(self.timeout)
             self.sock.connect((self.host, self.port))
 
-            if isinstance(self, compat_http_client.HTTPSConnection):
+            if isinstance(self, http.client.HTTPSConnection):
                 if hasattr(self, '_context'):  # Python > 2.6
                     self.sock = self._context.wrap_socket(
                         self.sock, server_hostname=self.host)
@@ -1440,10 +1435,10 @@ def make_socks_conn_class(base_class, socks_proxy):
     return SocksConnection
 
 
-class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
+class YoutubeDLHTTPSHandler(urllib.request.HTTPSHandler):
     def __init__(self, params, https_conn_class=None, *args, **kwargs):
-        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
-        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
+        urllib.request.HTTPSHandler.__init__(self, *args, **kwargs)
+        self._https_conn_class = https_conn_class or http.client.HTTPSConnection
         self._params = params
 
     def https_open(self, req):
@@ -1470,7 +1465,7 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             raise
 
 
-class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
+class YoutubeDLCookieJar(http.cookiejar.MozillaCookieJar):
     """
     See [1] for cookie file format.
 
@@ -1541,7 +1536,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
             if self.filename is not None:
                 filename = self.filename
             else:
-                raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
 
         # Store session cookies with `expires` set to 0 instead of an empty string
         for cookie in self:
@@ -1558,7 +1553,7 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
             if self.filename is not None:
                 filename = self.filename
             else:
-                raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
+                raise ValueError(http.cookiejar.MISSING_FILENAME_TEXT)
 
         def prepare_line(line):
             if line.startswith(self._HTTPONLY_PREFIX):
@@ -1568,10 +1563,10 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
                 return line
             cookie_list = line.split('\t')
             if len(cookie_list) != self._ENTRY_LEN:
-                raise compat_cookiejar.LoadError('invalid length %d' % len(cookie_list))
+                raise http.cookiejar.LoadError('invalid length %d' % len(cookie_list))
             cookie = self._CookieFileEntry(*cookie_list)
             if cookie.expires_at and not cookie.expires_at.isdigit():
-                raise compat_cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
+                raise http.cookiejar.LoadError('invalid expires at %s' % cookie.expires_at)
             return line
 
         cf = io.StringIO()
@@ -1579,9 +1574,9 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
             for line in f:
                 try:
                     cf.write(prepare_line(line))
-                except compat_cookiejar.LoadError as e:
+                except http.cookiejar.LoadError as e:
                     if f'{line.strip()} '[0] in '[{"':
-                        raise compat_cookiejar.LoadError(
+                        raise http.cookiejar.LoadError(
                             'Cookies file must be Netscape formatted, not JSON. See  '
                             'https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl')
                     write_string(f'WARNING: skipping cookie file entry due to {e}: {line!r}\n')
@@ -1604,18 +1599,18 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
                 cookie.discard = True
 
 
-class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
+class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
     def __init__(self, cookiejar=None):
-        compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
+        urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
 
     def http_response(self, request, response):
-        return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
+        return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
 
-    https_request = compat_urllib_request.HTTPCookieProcessor.http_request
+    https_request = urllib.request.HTTPCookieProcessor.http_request
     https_response = http_response
 
 
-class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
+class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
     """YoutubeDL redirect handler
 
     The code is based on HTTPRedirectHandler implementation from CPython [1].
@@ -1630,7 +1625,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
     3. https://github.com/ytdl-org/youtube-dl/issues/28768
     """
 
-    http_error_301 = http_error_303 = http_error_307 = http_error_308 = compat_urllib_request.HTTPRedirectHandler.http_error_302
+    http_error_301 = http_error_303 = http_error_307 = http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
 
     def redirect_request(self, req, fp, code, msg, headers, newurl):
         """Return a Request or None in response to a redirect.
@@ -1672,7 +1667,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         if code in (301, 302) and m == 'POST':
             m = 'GET'
 
-        return compat_urllib_request.Request(
+        return urllib.request.Request(
             newurl, headers=newheaders, origin_req_host=req.origin_req_host,
             unverifiable=True, method=m)
 
@@ -1967,7 +1962,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
-    return compat_struct_pack('%dB' % len(xs), *xs)
+    return struct.pack('%dB' % len(xs), *xs)
 
 
 class LockingUnsupportedError(OSError):
@@ -2427,12 +2422,12 @@ def urljoin(base, path):
     return compat_urlparse.urljoin(base, path)
 
 
-class HEADRequest(compat_urllib_request.Request):
+class HEADRequest(urllib.request.Request):
     def get_method(self):
         return 'HEAD'
 
 
-class PUTRequest(compat_urllib_request.Request):
+class PUTRequest(urllib.request.Request):
     def get_method(self):
         return 'PUT'
 
@@ -2484,7 +2479,7 @@ def url_or_none(url):
 
 
 def request_to_url(req):
-    if isinstance(req, compat_urllib_request.Request):
+    if isinstance(req, urllib.request.Request):
         return req.get_full_url()
     else:
         return req
@@ -3037,7 +3032,7 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     elif req_get_method == 'PUT':
         req_type = PUTRequest
     else:
-        req_type = compat_urllib_request.Request
+        req_type = urllib.request.Request
     new_req = req_type(
         req_url, data=req_data, headers=req_headers,
         origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
@@ -4636,20 +4631,20 @@ class GeoUtils:
         else:
             block = code_or_block
         addr, preflen = block.split('/')
-        addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
+        addr_min = struct.unpack('!L', socket.inet_aton(addr))[0]
         addr_max = addr_min | (0xffffffff >> int(preflen))
         return compat_str(socket.inet_ntoa(
-            compat_struct_pack('!L', random.randint(addr_min, addr_max))))
+            struct.pack('!L', random.randint(addr_min, addr_max))))
 
 
-class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
+class PerRequestProxyHandler(urllib.request.ProxyHandler):
     def __init__(self, proxies=None):
         # Set default handlers
         for type in ('http', 'https'):
             setattr(self, '%s_open' % type,
                     lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
                         meth(r, proxy, type))
-        compat_urllib_request.ProxyHandler.__init__(self, proxies)
+        urllib.request.ProxyHandler.__init__(self, proxies)
 
     def proxy_open(self, req, proxy, type):
         req_proxy = req.headers.get('Ytdl-request-proxy')
@@ -4663,7 +4658,7 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             req.add_header('Ytdl-socks-proxy', proxy)
             # yt-dlp's http/https handlers do wrapping the socket with socks
             return None
-        return compat_urllib_request.ProxyHandler.proxy_open(
+        return urllib.request.ProxyHandler.proxy_open(
             self, req, proxy, type)
 
 
@@ -4683,7 +4678,7 @@ def long_to_bytes(n, blocksize=0):
     s = b''
     n = int(n)
     while n > 0:
-        s = compat_struct_pack('>I', n & 0xffffffff) + s
+        s = struct.pack('>I', n & 0xffffffff) + s
         n = n >> 32
     # strip off leading zeros
     for i in range(len(s)):
@@ -4714,7 +4709,7 @@ def bytes_to_long(s):
         s = b'\000' * extra + s
         length = length + extra
     for i in range(0, length, 4):
-        acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
+        acc = (acc << 32) + struct.unpack('>I', s[i:i + 4])[0]
     return acc
 
 
@@ -4842,7 +4837,7 @@ def decode_png(png_data):
         raise OSError('Not a valid PNG file.')
 
     int_map = {1: '>B', 2: '>H', 4: '>I'}
-    unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
+    unpack_integer = lambda x: struct.unpack(int_map[len(x)], x)[0]
 
     chunks = []
 
@@ -4954,7 +4949,6 @@ def write_xattr(path, key, value):
         return
 
     # UNIX Method 1. Use xattrs/pyxattrs modules
-    from .dependencies import xattr
 
     setxattr = None
     if getattr(xattr, '_yt_dlp__identifier', None) == 'pyxattr':
-- 
cgit v1.2.3


From 54007a45f11ed730352324289b714baefd2901eb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 16:36:16 +0530
Subject: [cleanup] Consistent style for file heads

---
 yt_dlp/utils.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 6b02eb450..7327f3150 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 import atexit
 import base64
 import binascii
@@ -16,6 +15,8 @@ import hashlib
 import hmac
 import html.entities
 import html.parser
+import http.client
+import http.cookiejar
 import importlib.util
 import io
 import itertools
@@ -42,8 +43,6 @@ import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
 import zlib
-import http.client
-import http.cookiejar
 
 from .compat import asyncio, functools  # isort: split
 from .compat import (
-- 
cgit v1.2.3


From 14f25df2b6233553e968df023430ca96c0b1df9f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 16:24:43 +0530
Subject: [compat] Remove deprecated functions from core code

---
 yt_dlp/utils.py | 89 +++++++++++++++++++++++++++------------------------------
 1 file changed, 42 insertions(+), 47 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index 7327f3150..fd6c20682 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -39,6 +39,7 @@ import tempfile
 import time
 import traceback
 import types
+import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree
@@ -49,14 +50,8 @@ from .compat import (
     compat_etree_fromstring,
     compat_expanduser,
     compat_HTMLParseError,
-    compat_HTTPError,
     compat_os_name,
-    compat_parse_qs,
     compat_shlex_quote,
-    compat_str,
-    compat_urllib_parse_urlencode,
-    compat_urllib_parse_urlparse,
-    compat_urlparse,
 )
 from .dependencies import brotli, certifi, websockets, xattr
 from .socks import ProxyType, sockssocket
@@ -67,8 +62,8 @@ def register_socks_protocols():
     # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
     # URLs with protocols not in urlparse.uses_netloc are not handled correctly
     for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
-        if scheme not in compat_urlparse.uses_netloc:
-            compat_urlparse.uses_netloc.append(scheme)
+        if scheme not in urllib.parse.uses_netloc:
+            urllib.parse.uses_netloc.append(scheme)
 
 
 # This is not clearly defined otherwise
@@ -311,7 +306,7 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):
         return node.find(xpath)
 
-    if isinstance(xpath, (str, compat_str)):
+    if isinstance(xpath, str):
         n = _find_xpath(xpath)
     else:
         for xp in xpath:
@@ -741,10 +736,10 @@ def sanitize_url(url):
 
 
 def extract_basic_auth(url):
-    parts = compat_urlparse.urlsplit(url)
+    parts = urllib.parse.urlsplit(url)
     if parts.username is None:
         return url, None
-    url = compat_urlparse.urlunsplit(parts._replace(netloc=(
+    url = urllib.parse.urlunsplit(parts._replace(netloc=(
         parts.hostname if parts.port is None
         else '%s:%d' % (parts.hostname, parts.port))))
     auth_payload = base64.b64encode(
@@ -889,7 +884,7 @@ def decodeFilename(b, for_subprocess=False):
 def encodeArgument(s):
     # Legacy code that uses byte strings
     # Uncomment the following line after fixing all post processors
-    # assert isinstance(s, str), 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
+    # assert isinstance(s, str), 'Internal error: %r should be of type %r, is %r' % (s, str, type(s))
     return s if isinstance(s, str) else s.decode('ascii')
 
 
@@ -903,7 +898,7 @@ def decodeOption(optval):
     if isinstance(optval, bytes):
         optval = optval.decode(preferredencoding())
 
-    assert isinstance(optval, compat_str)
+    assert isinstance(optval, str)
     return optval
 
 
@@ -1395,7 +1390,7 @@ def make_socks_conn_class(base_class, socks_proxy):
     assert issubclass(base_class, (
         http.client.HTTPConnection, http.client.HTTPSConnection))
 
-    url_components = compat_urlparse.urlparse(socks_proxy)
+    url_components = urllib.parse.urlparse(socks_proxy)
     if url_components.scheme.lower() == 'socks5':
         socks_type = ProxyType.SOCKS5
     elif url_components.scheme.lower() in ('socks', 'socks4'):
@@ -1639,7 +1634,7 @@ class YoutubeDLRedirectHandler(urllib.request.HTTPRedirectHandler):
         m = req.get_method()
         if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
                  or code in (301, 302, 303) and m == "POST")):
-            raise compat_HTTPError(req.full_url, code, msg, headers, fp)
+            raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp)
         # Strictly (according to RFC 2616), 301 or 302 in response to
         # a POST MUST NOT cause a redirection without confirmation
         # from the user (of urllib.request, in this case).  In practice,
@@ -1739,7 +1734,7 @@ def unified_strdate(date_str, day_first=True):
             with contextlib.suppress(ValueError):
                 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     if upload_date is not None:
-        return compat_str(upload_date)
+        return str(upload_date)
 
 
 def unified_timestamp(date_str, day_first=True):
@@ -1913,12 +1908,12 @@ class DateRange:
 
 
 def platform_name():
-    """ Returns the platform name as a compat_str """
+    """ Returns the platform name as a str """
     res = platform.platform()
     if isinstance(res, bytes):
         res = res.decode(preferredencoding())
 
-    assert isinstance(res, compat_str)
+    assert isinstance(res, str)
     return res
 
 
@@ -2144,7 +2139,7 @@ def smuggle_url(url, data):
 
     url, idata = unsmuggle_url(url, {})
     data.update(idata)
-    sdata = compat_urllib_parse_urlencode(
+    sdata = urllib.parse.urlencode(
         {'__youtubedl_smuggle': json.dumps(data)})
     return url + '#' + sdata
 
@@ -2153,7 +2148,7 @@ def unsmuggle_url(smug_url, default=None):
     if '#__youtubedl_smuggle' not in smug_url:
         return smug_url, default
     url, _, sdata = smug_url.rpartition('#')
-    jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
+    jsond = urllib.parse.parse_qs(sdata)['__youtubedl_smuggle'][0]
     data = json.loads(jsond)
     return url, data
 
@@ -2313,7 +2308,7 @@ def parse_resolution(s, *, lenient=False):
 
 
 def parse_bitrate(s):
-    if not isinstance(s, compat_str):
+    if not isinstance(s, str):
         return
     mobj = re.search(r'\b(\d+)\s*kbps', s)
     if mobj:
@@ -2350,7 +2345,7 @@ def fix_xml_ampersands(xml_str):
 
 
 def setproctitle(title):
-    assert isinstance(title, compat_str)
+    assert isinstance(title, str)
 
     # ctypes in Jython is not complete
     # http://bugs.jython.org/issue2148
@@ -2398,7 +2393,7 @@ def get_domain(url):
 
 
 def url_basename(url):
-    path = compat_urlparse.urlparse(url).path
+    path = urllib.parse.urlparse(url).path
     return path.strip('/').split('/')[-1]
 
 
@@ -2409,16 +2404,16 @@ def base_url(url):
 def urljoin(base, path):
     if isinstance(path, bytes):
         path = path.decode()
-    if not isinstance(path, compat_str) or not path:
+    if not isinstance(path, str) or not path:
         return None
     if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
         return path
     if isinstance(base, bytes):
         base = base.decode()
-    if not isinstance(base, compat_str) or not re.match(
+    if not isinstance(base, str) or not re.match(
             r'^(?:https?:)?//', base):
         return None
-    return compat_urlparse.urljoin(base, path)
+    return urllib.parse.urljoin(base, path)
 
 
 class HEADRequest(urllib.request.Request):
@@ -2441,14 +2436,14 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
 
 
 def str_or_none(v, default=None):
-    return default if v is None else compat_str(v)
+    return default if v is None else str(v)
 
 
 def str_to_int(int_str):
     """ A more relaxed version of int_or_none """
     if isinstance(int_str, int):
         return int_str
-    elif isinstance(int_str, compat_str):
+    elif isinstance(int_str, str):
         int_str = re.sub(r'[,\.\+]', '', int_str)
         return int_or_none(int_str)
 
@@ -2467,11 +2462,11 @@ def bool_or_none(v, default=None):
 
 
 def strip_or_none(v, default=None):
-    return v.strip() if isinstance(v, compat_str) else default
+    return v.strip() if isinstance(v, str) else default
 
 
 def url_or_none(url):
-    if not url or not isinstance(url, compat_str):
+    if not url or not isinstance(url, str):
         return None
     url = url.strip()
     return url if re.match(r'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
@@ -2489,7 +2484,7 @@ def strftime_or_none(timestamp, date_format, default=None):
     try:
         if isinstance(timestamp, (int, float)):  # unix timestamp
             datetime_object = datetime.datetime.utcfromtimestamp(timestamp)
-        elif isinstance(timestamp, compat_str):  # assume YYYYMMDD
+        elif isinstance(timestamp, str):  # assume YYYYMMDD
             datetime_object = datetime.datetime.strptime(timestamp, '%Y%m%d')
         return datetime_object.strftime(date_format)
     except (ValueError, TypeError, AttributeError):
@@ -2592,7 +2587,7 @@ def _get_exe_version_output(exe, args, *, to_screen=None):
 
 
 def detect_exe_version(output, version_re=None, unrecognized='present'):
-    assert isinstance(output, compat_str)
+    assert isinstance(output, str)
     if version_re is None:
         version_re = r'version\s+([-0-9._a-zA-Z]+)'
     m = re.search(version_re, output)
@@ -2973,7 +2968,7 @@ def escape_rfc3986(s):
 
 def escape_url(url):
     """Escape URL as suggested by RFC 3986"""
-    url_parsed = compat_urllib_parse_urlparse(url)
+    url_parsed = urllib.parse.urlparse(url)
     return url_parsed._replace(
         netloc=url_parsed.netloc.encode('idna').decode('ascii'),
         path=escape_rfc3986(url_parsed.path),
@@ -2984,12 +2979,12 @@ def escape_url(url):
 
 
 def parse_qs(url):
-    return compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+    return urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
 
 
 def read_batch_urls(batch_fd):
     def fixup(url):
-        if not isinstance(url, compat_str):
+        if not isinstance(url, str):
             url = url.decode('utf-8', 'replace')
         BOM_UTF8 = ('\xef\xbb\xbf', '\ufeff')
         for bom in BOM_UTF8:
@@ -3007,17 +3002,17 @@ def read_batch_urls(batch_fd):
 
 
 def urlencode_postdata(*args, **kargs):
-    return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
+    return urllib.parse.urlencode(*args, **kargs).encode('ascii')
 
 
 def update_url_query(url, query):
     if not query:
         return url
-    parsed_url = compat_urlparse.urlparse(url)
-    qs = compat_parse_qs(parsed_url.query)
+    parsed_url = urllib.parse.urlparse(url)
+    qs = urllib.parse.parse_qs(parsed_url.query)
     qs.update(query)
-    return compat_urlparse.urlunparse(parsed_url._replace(
-        query=compat_urllib_parse_urlencode(qs, True)))
+    return urllib.parse.urlunparse(parsed_url._replace(
+        query=urllib.parse.urlencode(qs, True)))
 
 
 def update_Request(req, url=None, data=None, headers={}, query={}):
@@ -3046,9 +3041,9 @@ def _multipart_encode_impl(data, boundary):
     out = b''
     for k, v in data.items():
         out += b'--' + boundary.encode('ascii') + b'\r\n'
-        if isinstance(k, compat_str):
+        if isinstance(k, str):
             k = k.encode()
-        if isinstance(v, compat_str):
+        if isinstance(v, str):
             v = v.encode()
         # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
         # suggests sending UTF-8 directly. Firefox sends UTF-8, too
@@ -3129,7 +3124,7 @@ def merge_dicts(*dicts):
 
 
 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
-    return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
+    return string if isinstance(string, str) else str(string, encoding, errors)
 
 
 US_RATINGS = {
@@ -3509,7 +3504,7 @@ def determine_protocol(info_dict):
     elif ext == 'f4m':
         return 'f4m'
 
-    return compat_urllib_parse_urlparse(url).scheme
+    return urllib.parse.urlparse(url).scheme
 
 
 def render_table(header_row, data, delim=False, extra_gap=0, hide_empty=False):
@@ -4632,7 +4627,7 @@ class GeoUtils:
         addr, preflen = block.split('/')
         addr_min = struct.unpack('!L', socket.inet_aton(addr))[0]
         addr_max = addr_min | (0xffffffff >> int(preflen))
-        return compat_str(socket.inet_ntoa(
+        return str(socket.inet_ntoa(
             struct.pack('!L', random.randint(addr_min, addr_max))))
 
 
@@ -4653,7 +4648,7 @@ class PerRequestProxyHandler(urllib.request.ProxyHandler):
 
         if proxy == '__noproxy__':
             return None  # No Proxy
-        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if urllib.parse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
             req.add_header('Ytdl-socks-proxy', proxy)
             # yt-dlp's http/https handlers do wrapping the socket with socks
             return None
@@ -5036,7 +5031,7 @@ def iri_to_uri(iri):
     The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact.
     """
 
-    iri_parts = compat_urllib_parse_urlparse(iri)
+    iri_parts = urllib.parse.urlparse(iri)
 
     if '[' in iri_parts.netloc:
         raise ValueError('IPv6 URIs are not, yet, supported.')
-- 
cgit v1.2.3


From c043c246251da815c99f8c779194fcdef9ef7a58 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 25 Jun 2022 19:41:22 +0530
Subject: [extractor] Fix `_create_request` when headers is None

Closes #4164
---
 yt_dlp/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'yt_dlp/utils.py')

diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py
index fd6c20682..46a6c9fce 100644
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@@ -3015,9 +3015,9 @@ def update_url_query(url, query):
         query=urllib.parse.urlencode(qs, True)))
 
 
-def update_Request(req, url=None, data=None, headers={}, query={}):
+def update_Request(req, url=None, data=None, headers=None, query=None):
     req_headers = req.headers.copy()
-    req_headers.update(headers)
+    req_headers.update(headers or {})
     req_data = data or req.data
     req_url = update_url_query(url or req.get_full_url(), query)
     req_get_method = req.get_method()
-- 
cgit v1.2.3