16 files changed, 88 insertions, 60 deletions
diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index 12abd45..e0768d2 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import os
@@ -8,7 +8,7 @@ import sys
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import hypervideo_dl
 
-BASH_COMPLETION_FILE = "hypervideo.bash-completion"
+BASH_COMPLETION_FILE = "completions/bash/hypervideo"
 BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
 
 
diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index 6f8aae1..2a8039e 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -1,3 +1,5 @@
+# UNUSED
+
 #!/usr/bin/python3
 
 import argparse
diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index a62711e..7dd372f 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 """
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index b6e9949..84ced2d 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import optparse
@@ -10,10 +10,11 @@ sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import hypervideo_dl
 from hypervideo_dl.utils import shell_quote
 
-FISH_COMPLETION_FILE = 'hypervideo.fish'
+FISH_COMPLETION_FILE = 'completions/fish/hypervideo.fish'
 FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
 
 EXTRA_ARGS = {
+    'remux-video': ['--arguments', 'mp4 mkv', '--exclusive'],
     'recode-video': ['--arguments', 'mp4 flv ogg webm mkv', '--exclusive'],
 
     # Options that need a file parameter
diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
index 00dc5bd..09feeaa 100644
--- a/devscripts/generate_aes_testdata.py
+++ b/devscripts/generate_aes_testdata.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import codecs
diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py
index c4e5fc1..da89e07 100644
--- a/devscripts/lazy_load_template.py
+++ b/devscripts/lazy_load_template.py
@@ -1,19 +1,31 @@
 # coding: utf-8
-from __future__ import unicode_literals
-
 import re
 
+from ..utils import bug_reports_message, write_string
+
+
+class LazyLoadMetaClass(type):
+    def __getattr__(cls, name):
+        if '_real_class' not in cls.__dict__:
+            write_string(
+                f'WARNING: Falling back to normal extractor since lazy extractor '
+                f'{cls.__name__} does not have attribute {name}{bug_reports_message()}')
+        return getattr(cls._get_real_class(), name)
+
 
-class LazyLoadExtractor(object):
+class LazyLoadExtractor(metaclass=LazyLoadMetaClass):
     _module = None
+    _WORKING = True
 
     @classmethod
-    def ie_key(cls):
-        return cls.__name__[:-2]
+    def _get_real_class(cls):
+        if '_real_class' not in cls.__dict__:
+            mod = __import__(cls._module, fromlist=(cls.__name__,))
+            cls._real_class = getattr(mod, cls.__name__)
+        return cls._real_class
 
     def __new__(cls, *args, **kwargs):
-        mod = __import__(cls._module, fromlist=(cls.__name__,))
-        real_cls = getattr(mod, cls.__name__)
+        real_cls = cls._get_real_class()
         instance = real_cls.__new__(real_cls)
         instance.__init__(*args, **kwargs)
         return instance
diff --git a/devscripts/logo.ico b/devscripts/logo.ico
new file mode 100644
index 0000000..5503a43
--- /dev/null
+++ b/devscripts/logo.ico
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index dbc2e08..8c5f107 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import io
@@ -7,6 +7,8 @@ import re
 
 
 def main():
+    return  # This is unused in hypervideo
+
     parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
     options, args = parser.parse_args()
     if len(args) != 2:
@@ -20,8 +22,7 @@ def main():
     bug_text = re.search(
         r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
     dev_text = re.search(
-        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING HYPERVIDEO',
-        readme).group(1)
+        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING HYPERVIDEO', readme).group(1)
 
     out = bug_text + dev_text
 
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index b9a851c..7a38e40 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 from __future__ import unicode_literals, print_function
 
 from inspect import getsource
@@ -6,27 +7,35 @@ import os
 from os.path import dirname as dirn
 import sys
 
-print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
-
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 
 lazy_extractors_filename = sys.argv[1]
 if os.path.exists(lazy_extractors_filename):
     os.remove(lazy_extractors_filename)
 
+# Block plugins from loading
+plugins_dirname = 'ytdlp_plugins'
+plugins_blocked_dirname = 'ytdlp_plugins_blocked'
+if os.path.exists(plugins_dirname):
+    os.rename(plugins_dirname, plugins_blocked_dirname)
+
 from hypervideo_dl.extractor import _ALL_CLASSES
 from hypervideo_dl.extractor.common import InfoExtractor, SearchInfoExtractor
 
+if os.path.exists(plugins_blocked_dirname):
+    os.rename(plugins_blocked_dirname, plugins_dirname)
+
 with open('devscripts/lazy_load_template.py', 'rt') as f:
     module_template = f.read()
 
+CLASS_PROPERTIES = ['ie_key', 'working', '_match_valid_url', 'suitable', '_match_id', 'get_temp_id']
 module_contents = [
-    module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
-    'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']
+    module_template,
+    *[getsource(getattr(InfoExtractor, k)) for k in CLASS_PROPERTIES],
+    '\nclass LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']
 
 ie_template = '''
 class {name}({bases}):
-    _VALID_URL = {valid_url!r}
     _module = '{module}'
 '''
 
@@ -47,14 +56,17 @@ def get_base_name(base):
 
 
 def build_lazy_ie(ie, name):
-    valid_url = getattr(ie, '_VALID_URL', None)
     s = ie_template.format(
         name=name,
         bases=', '.join(map(get_base_name, ie.__bases__)),
-        valid_url=valid_url,
         module=ie.__module__)
+    valid_url = getattr(ie, '_VALID_URL', None)
+    if valid_url:
+        s += f'    _VALID_URL = {valid_url!r}\n'
+    if not ie._WORKING:
+        s += '    _WORKING = False\n'
     if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
-        s += '\n' + getsource(ie.suitable)
+        s += f'\n{getsource(ie.suitable)}'
     if hasattr(ie, '_make_valid_url'):
         # search extractors
         s += make_valid_template.format(valid_url=ie._make_valid_url())
@@ -92,7 +104,7 @@ for ie in ordered_cls:
         names.append(name)
 
 module_contents.append(
-    '_ALL_CLASSES = [{0}]'.format(', '.join(names)))
+    '\n_ALL_CLASSES = [{0}]'.format(', '.join(names)))
 
 module_src = '\n'.join(module_contents) + '\n'
 
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 8fbce07..1a9a017 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -1,3 +1,8 @@
+#!/usr/bin/env python3
+
+# hypervideo --help | make_readme.py
+# This must be run in a console of correct width
+
 from __future__ import unicode_literals
 
 import io
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 09807b0..a079406 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import io
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 5b74238..58090d4 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import io
diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat
index 01a79b6..b8bb393 100644
--- a/devscripts/run_tests.bat
+++ b/devscripts/run_tests.bat
@@ -1,17 +1,16 @@
+@setlocal
 @echo off
+cd /d %~dp0..
 
-rem Keep this list in sync with the `offlinetest` target in Makefile
-set DOWNLOAD_TESTS="age_restriction^|download^|socks^|subtitles^|write_annotations^|youtube_lists^|youtube_signature"
-
-if "%YTDL_TEST_SET%" == "core" (
-    set test_set="-I test_("%DOWNLOAD_TESTS%")\.py"
-    set multiprocess_args=""
-) else if "%YTDL_TEST_SET%" == "download" (
-    set test_set="-I test_(?!"%DOWNLOAD_TESTS%").+\.py"
-    set multiprocess_args="--processes=4 --process-timeout=540"
+if ["%~1"]==[""] (
+    set "test_set="test""
+) else if ["%~1"]==["core"] (
+    set "test_set="-m not download""
+) else if ["%~1"]==["download"] (
+    set "test_set="-m "download""
 ) else (
-    echo YTDL_TEST_SET is not set or invalid
+    echo.Invalid test type "%~1". Use "core" ^| "download"
     exit /b 1
 )
 
-nosetests test --verbose %test_set:"=% %multiprocess_args:"=%
+pytest %test_set%
diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh
index b8f48b9..c9a75ba 100755
--- a/devscripts/run_tests.sh
+++ b/devscripts/run_tests.sh
@@ -1,22 +1,14 @@
-#!/bin/bash
-
-# Keep this list in sync with the `offlinetest` target in Makefile
-DOWNLOAD_TESTS="age_restriction|download|socks|subtitles|write_annotations|youtube_lists|youtube_signature"
-
-test_set=""
-multiprocess_args=""
-
-case "$YTDL_TEST_SET" in
-    core)
-        test_set="-I test_($DOWNLOAD_TESTS)\.py"
-    ;;
-    download)
-        test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py"
-        multiprocess_args="--processes=4 --process-timeout=540"
-    ;;
-    *)
-        break
-    ;;
-esac
-
-nosetests test --verbose $test_set $multiprocess_args
+#!/bin/sh
+
+if [ -z $1 ]; then
+    test_set='test'
+elif [ $1 = 'core' ]; then
+    test_set="-m not download"
+elif [ $1 = 'download' ]; then
+    test_set="-m download"
+else
+    echo 'Invalid test type "'$1'". Use "core" | "download"'
+    exit 1
+fi
+
+python3 -m pytest "$test_set"
diff --git a/devscripts/zsh-completion.in b/devscripts/zsh-completion.in
index 1906949..e5cb92e 100644
--- a/devscripts/zsh-completion.in
+++ b/devscripts/zsh-completion.in
@@ -16,6 +16,8 @@ __hypervideo_dl() {
                 _path_files
             elif [[ ${prev} =~ ${diropts} ]]; then
                 _path_files -/
+            elif [[ ${prev} == "--remux-video" ]]; then
+                _arguments '*: :(mp4 mkv)'
             elif [[ ${prev} == "--recode-video" ]]; then
                 _arguments '*: :(mp4 flv ogg webm mkv)'
             else
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index b570469..c8620a5 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 from __future__ import unicode_literals
 
 import os
@@ -8,7 +8,7 @@ import sys
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import hypervideo_dl
 
-ZSH_COMPLETION_FILE = "hypervideo.zsh"
+ZSH_COMPLETION_FILE = "completions/zsh/_hypervideo"
 ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"