From 21633673c33f082c6673bc245e4a90d880729a58 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 18 May 2022 09:04:30 +0530
Subject: [cleanup] Minor fixes

---
 devscripts/make_lazy_extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devscripts')

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 8c481bc2d..cd1985c8e 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
-import os
 import optparse
+import os
 import sys
 from inspect import getsource
 
-- 
cgit v1.2.3


From c487cf00101525ff836d59a2a42ef63e85ea9556 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 17 Apr 2022 22:48:50 +0530
Subject: [cleanup] Misc

---
 devscripts/make_issue_template.py |  1 -
 devscripts/make_readme.py         | 46 ++++++++++++++++++++++++++++++---------
 2 files changed, 36 insertions(+), 11 deletions(-)

(limited to 'devscripts')

diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 811a3e9b5..5a309008e 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import io
 import optparse
 
 
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index fd234bf58..15c4a7c7d 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -2,6 +2,7 @@
 
 # yt-dlp --help | make_readme.py
 # This must be run in a console of correct width
+import functools
 import re
 import sys
 
@@ -12,19 +13,44 @@ OPTIONS_END = 'CONFIGURATION'
 EPILOG_START = 'See full documentation'
 
 
-helptext = sys.stdin.read()
-if isinstance(helptext, bytes):
-    helptext = helptext.decode()
+def take_section(text, start=None, end=None, *, shift=0):
+    return text[
+        text.index(start) + shift if start else None:
+        text.index(end) + shift if end else None
+    ]
 
-start, end = helptext.index(f'\n  {OPTIONS_START}'), helptext.index(f'\n{EPILOG_START}')
-options = re.sub(r'(?m)^  (\w.+)$', r'## \1', helptext[start + 1: end + 1])
+
+def apply_patch(text, patch):
+    return re.sub(*patch, text)
+
+
+options = take_section(sys.stdin.read(), f'\n  {OPTIONS_START}', f'\n{EPILOG_START}', shift=1)
+
+switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group())
+delim = f'\n{" " * switch_col_width}'
+
+PATCHES = (
+    (  # Headings
+        r'(?m)^  (\w.+\n)(    (?=\w))?',
+        r'## \1'
+    ),
+    (  # Do not split URLs
+        rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
+        lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
+    ),
+    # This creates issues with prepare_manpage
+    # (  # Avoid newline when a space is available b/w switch and description
+    #     r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
+    #     r'\1 '
+    # ),
+)
 
 with open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
-header = readme[:readme.index(f'## {OPTIONS_START}')]
-footer = readme[readme.index(f'# {OPTIONS_END}'):]
-
 with open(README_FILE, 'w', encoding='utf-8') as f:
-    for part in (header, options, footer):
-        f.write(part)
+    f.write(''.join((
+        take_section(readme, end=f'## {OPTIONS_START}'),
+        functools.reduce(apply_patch, PATCHES, options),
+        take_section(readme, f'# {OPTIONS_END}'),
+    )))
-- 
cgit v1.2.3


From 8a82af3511b4379af0d239dbd01c672c17a2c46a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 27 May 2022 04:36:23 +0530
Subject: [cleanup] Misc fixes and cleanup

Closes #3780, Closes #3853, Closes #3850
---
 devscripts/make_readme.py | 18 ++++++++++++------
 devscripts/run_tests.sh   |  2 +-
 2 files changed, 13 insertions(+), 7 deletions(-)

(limited to 'devscripts')

diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 15c4a7c7d..42578cb0a 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -12,6 +12,8 @@ OPTIONS_START = 'General Options:'
 OPTIONS_END = 'CONFIGURATION'
 EPILOG_START = 'See full documentation'
 
+DISABLE_PATCH = object()
+
 
 def take_section(text, start=None, end=None, *, shift=0):
     return text[
@@ -21,7 +23,7 @@ def take_section(text, start=None, end=None, *, shift=0):
 
 
 def apply_patch(text, patch):
-    return re.sub(*patch, text)
+    return text if patch[0] is DISABLE_PATCH else re.sub(*patch, text)
 
 
 options = take_section(sys.stdin.read(), f'\n  {OPTIONS_START}', f'\n{EPILOG_START}', shift=1)
@@ -38,11 +40,15 @@ PATCHES = (
         rf'({delim[:-1]})? (?P<label>\[\S+\] )?(?P<url>https?({delim})?:({delim})?/({delim})?/(({delim})?\S+)+)\s',
         lambda mobj: ''.join((delim, mobj.group('label') or '', re.sub(r'\s+', '', mobj.group('url')), '\n'))
     ),
-    # This creates issues with prepare_manpage
-    # (  # Avoid newline when a space is available b/w switch and description
-    #     r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
-    #     r'\1 '
-    # ),
+    (  # Do not split "words"
+        rf'(?m)({delim}\S+)+$',
+        lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
+    ),
+    (  # Avoid newline when a space is available b/w switch and description
+        DISABLE_PATCH,  # This creates issues with prepare_manpage
+        r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
+        r'\1 '
+    ),
 )
 
 with open(README_FILE, encoding='utf-8') as f:
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index e9904ae35..d496a092b 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env sh
 
 if [ -z $1 ]; then
     test_set='test'
-- 
cgit v1.2.3


From 64fa820ccf61a7aea6c2a48b1362b3a4ec270cad Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 25 May 2022 17:53:46 +0530
Subject: [cleanup] Misc fixes (see desc)

* [tvver] Fix bug in 6837633a4a614920b6e43ffc6b4b8590dca8c9d7 - Closes #4054
* [rumble] Fix tests - Closes #3976
* [make] Remove `cat` abuse - Closes #3989
* [make] Revert #3684 - Closes #3814
* [utils] Improve `get_elements_by_class` - Closes #3993
* [utils] Inherit `Namespace` from `types.SimpleNamespace`
* [utils] Use `re.fullmatch` for matching filters
* [jsinterp] Handle quotes in `_separate`
* [make_readme] Allow overshooting last line

Authored by: pukkandan, kwconder, MrRawes, Lesmiscore
---
 devscripts/make_readme.py | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'devscripts')

diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 42578cb0a..015212aa3 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -11,6 +11,7 @@ README_FILE = 'README.md'
 OPTIONS_START = 'General Options:'
 OPTIONS_END = 'CONFIGURATION'
 EPILOG_START = 'See full documentation'
+ALLOWED_OVERSHOOT = 2
 
 DISABLE_PATCH = object()
 
@@ -28,6 +29,7 @@ def apply_patch(text, patch):
 
 options = take_section(sys.stdin.read(), f'\n  {OPTIONS_START}', f'\n{EPILOG_START}', shift=1)
 
+max_width = max(map(len, options.split('\n')))
 switch_col_width = len(re.search(r'(?m)^\s{5,}', options).group())
 delim = f'\n{" " * switch_col_width}'
 
@@ -44,6 +46,12 @@ PATCHES = (
         rf'(?m)({delim}\S+)+$',
         lambda mobj: ''.join((delim, mobj.group(0).replace(delim, '')))
     ),
+    (  # Allow overshooting last line
+        rf'(?m)^(?P<prev>.+)${delim}(?P<current>.+)$(?!{delim})',
+        lambda mobj: (mobj.group().replace(delim, ' ')
+                      if len(mobj.group()) - len(delim) + 1 <= max_width + ALLOWED_OVERSHOOT
+                      else mobj.group())
+    ),
     (  # Avoid newline when a space is available b/w switch and description
         DISABLE_PATCH,  # This creates issues with prepare_manpage
         r'(?m)^(\s{4}-.{%d})(%s)' % (switch_col_width - 6, delim),
-- 
cgit v1.2.3


From 560738f34de4df6eaf82290fd503def3f366f878 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 15 Jun 2022 18:00:34 +0530
Subject: [extractor] Import `_ALL_CLASSES` lazily

This significantly speeds up `import yt_dlp` in the absence of `lazy_extractors`
---
 devscripts/make_lazy_extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devscripts')

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index cd1985c8e..39d4646d0 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -53,7 +53,7 @@ def get_all_ies():
     if os.path.exists(PLUGINS_DIRNAME):
         os.rename(PLUGINS_DIRNAME, BLOCKED_DIRNAME)
     try:
-        from yt_dlp.extractor import _ALL_CLASSES
+        from yt_dlp.extractor.extractors import _ALL_CLASSES
     finally:
         if os.path.exists(BLOCKED_DIRNAME):
             os.rename(BLOCKED_DIRNAME, PLUGINS_DIRNAME)
-- 
cgit v1.2.3


From ac668111128b5f124b4271b3aa4c35f6e71a4749 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 13:40:17 +0530
Subject: [compat] Remove more functions

Removing any more will require changes to a large number of extractors
---
 devscripts/check-porn.py      |  8 +++++---
 devscripts/update-formulae.py | 11 +++++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'devscripts')

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 08f663e4b..89644a43d 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -13,9 +13,11 @@ import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import gettestcases
 
-from yt_dlp.utils import compat_urllib_parse_urlparse, compat_urllib_request
+import urllib.request
+
+from test.helper import gettestcases
+from yt_dlp.utils import compat_urllib_parse_urlparse
 
 if len(sys.argv) > 1:
     METHOD = 'LIST'
@@ -26,7 +28,7 @@ else:
 for test in gettestcases():
     if METHOD == 'EURISTIC':
         try:
-            webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
+            webpage = urllib.request.urlopen(test['url'], timeout=10).read()
         except Exception:
             print('\nFail: {}'.format(test['name']))
             continue
diff --git a/devscripts/update-formulae.py b/devscripts/update-formulae.py
index a89872c7b..02b869304 100644
--- a/devscripts/update-formulae.py
+++ b/devscripts/update-formulae.py
@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
-import json
+
+# Allow direct execution
 import os
-import re
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from yt_dlp.compat import compat_urllib_request
+
+import json
+import re
+import urllib.request
 
 # usage: python3 ./devscripts/update-formulae.py <path-to-formulae-rb> <version>
 # version can be either 0-aligned (yt-dlp version) or normalized (PyPl version)
@@ -15,7 +18,7 @@ filename, version = sys.argv[1:]
 
 normalized_version = '.'.join(str(int(x)) for x in version.split('.'))
 
-pypi_release = json.loads(compat_urllib_request.urlopen(
+pypi_release = json.loads(urllib.request.urlopen(
     'https://pypi.org/pypi/yt-dlp/%s/json' % normalized_version
 ).read().decode())
 
-- 
cgit v1.2.3


From 54007a45f11ed730352324289b714baefd2901eb Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 16:36:16 +0530
Subject: [cleanup] Consistent style for file heads

---
 devscripts/bash-completion.py       |  3 +++
 devscripts/fish-completion.py       |  6 +++++-
 devscripts/generate_aes_testdata.py |  8 ++++++--
 devscripts/make_contributing.py     |  1 +
 devscripts/make_issue_template.py   | 10 +++++++++-
 devscripts/make_lazy_extractors.py  |  7 +++++--
 devscripts/make_readme.py           |  8 ++++++--
 devscripts/make_supportedsites.py   |  6 +++++-
 devscripts/prepare_manpage.py       |  3 ++-
 devscripts/update-version.py        |  8 ++++++++
 devscripts/zsh-completion.py        |  3 +++
 11 files changed, 53 insertions(+), 10 deletions(-)

(limited to 'devscripts')

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index 268e8a2ae..9b4a9d4e2 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
+
+# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+
 import yt_dlp
 
 BASH_COMPLETION_FILE = "completions/bash/yt-dlp"
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index d9c0048e2..5d2f68a48 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -1,10 +1,14 @@
 #!/usr/bin/env python3
-import optparse
+
+# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+
+import optparse
+
 import yt_dlp
 from yt_dlp.utils import shell_quote
 
diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
index c7d83f1a7..7f3c88bcf 100644
--- a/devscripts/generate_aes_testdata.py
+++ b/devscripts/generate_aes_testdata.py
@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
-import codecs
+
+# Allow direct execution
 import os
-import subprocess
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+
+import codecs
+import subprocess
+
 from yt_dlp.aes import aes_encrypt, key_expansion
 from yt_dlp.utils import intlist_to_bytes
 
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 2562c4fd7..a06f8a616 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 import optparse
 import re
 
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index 5a309008e..54043ef4e 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -1,4 +1,12 @@
 #!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
 import optparse
 
 
@@ -7,7 +15,7 @@ def read(fname):
         return f.read()
 
 
-# Get the version from yt_dlp/version.py without importing the package
+# Get the version without importing the package
 def read_version(fname):
     exec(compile(read(fname), fname, 'exec'))
     return locals()['__version__']
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 39d4646d0..785d66a6a 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
-import optparse
+
+# Allow direct execution
 import os
 import sys
-from inspect import getsource
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import optparse
+from inspect import getsource
+
 NO_ATTR = object()
 STATIC_CLASS_PROPERTIES = ['IE_NAME', 'IE_DESC', 'SEARCH_KEY', '_WORKING', '_NETRC_MACHINE', 'age_limit']
 CLASS_METHODS = [
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 015212aa3..f2e08d7c6 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -1,7 +1,11 @@
 #!/usr/bin/env python3
 
-# yt-dlp --help | make_readme.py
-# This must be run in a console of correct width
+"""
+yt-dlp --help | make_readme.py
+This must be run in a console of correct width
+"""
+
+
 import functools
 import re
 import sys
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index d8c53c5e1..e46f7af56 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -1,10 +1,14 @@
 #!/usr/bin/env python3
-import optparse
+
+# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+
+import optparse
+
 from yt_dlp.extractor import list_extractor_classes
 
 
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 91e9ebced..cea934949 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 import optparse
 import os.path
 import re
@@ -23,7 +24,7 @@ yt\-dlp \- A youtube-dl fork with additional features and patches
 
 def main():
     parser = optparse.OptionParser(usage='%prog OUTFILE.md')
-    options, args = parser.parse_args()
+    _, args = parser.parse_args()
     if len(args) != 1:
         parser.error('Expected an output filename')
 
diff --git a/devscripts/update-version.py b/devscripts/update-version.py
index 991cfb2af..c5bc83de9 100644
--- a/devscripts/update-version.py
+++ b/devscripts/update-version.py
@@ -1,4 +1,12 @@
 #!/usr/bin/env python3
+
+# Allow direct execution
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
 import subprocess
 import sys
 from datetime import datetime
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index 59faea06a..267af5f6e 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -1,9 +1,12 @@
 #!/usr/bin/env python3
+
+# Allow direct execution
 import os
 import sys
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+
 import yt_dlp
 
 ZSH_COMPLETION_FILE = "completions/zsh/_yt-dlp"
-- 
cgit v1.2.3


From 14f25df2b6233553e968df023430ca96c0b1df9f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 24 Jun 2022 16:24:43 +0530
Subject: [compat] Remove deprecated functions from core code

---
 devscripts/check-porn.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'devscripts')

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 89644a43d..fc72c3051 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -14,10 +14,10 @@ import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
+import urllib.parse
 import urllib.request
 
 from test.helper import gettestcases
-from yt_dlp.utils import compat_urllib_parse_urlparse
 
 if len(sys.argv) > 1:
     METHOD = 'LIST'
@@ -38,7 +38,7 @@ for test in gettestcases():
         RESULT = 'porn' in webpage.lower()
 
     elif METHOD == 'LIST':
-        domain = compat_urllib_parse_urlparse(test['url']).netloc
+        domain = urllib.parse.urlparse(test['url']).netloc
         if not domain:
             print('\nFail: {}'.format(test['name']))
             continue
-- 
cgit v1.2.3