diff options
Diffstat (limited to 'test')
45 files changed, 1686 insertions, 941 deletions
diff --git a/test/helper.py b/test/helper.py index 1f1ccfa..1dae86f 100644 --- a/test/helper.py +++ b/test/helper.py @@ -1,26 +1,16 @@ -from __future__ import unicode_literals - import errno -import io import hashlib import json import os.path import re -import types import ssl import sys +import types import hypervideo_dl.extractor from hypervideo_dl import YoutubeDL -from hypervideo_dl.compat import ( - compat_os_name, - compat_str, -) -from hypervideo_dl.utils import ( - preferredencoding, - write_string, -) - +from hypervideo_dl.compat import compat_os_name +from hypervideo_dl.utils import preferredencoding, write_string if 'pytest' in sys.modules: import pytest @@ -35,10 +25,10 @@ def get_params(override=None): 'parameters.json') LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'local_parameters.json') - with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: + with open(PARAMETERS_FILE, encoding='utf-8') as pf: parameters = json.load(pf) if os.path.exists(LOCAL_PARAMETERS_FILE): - with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: + with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: parameters.update(json.load(pf)) if override: parameters.update(override) @@ -54,7 +44,7 @@ def try_rm(filename): raise -def report_warning(message): +def report_warning(message, *args, **kwargs): ''' Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored @@ -63,8 +53,8 @@ def report_warning(message): _msg_header = '\033[0;33mWARNING:\033[0m' else: _msg_header = 'WARNING:' - output = '%s %s\n' % (_msg_header, message) - if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3: + output = f'{_msg_header} {message}\n' + if 'b' in getattr(sys.stderr, 'mode', ''): output = output.encode(preferredencoding()) sys.stderr.write(output) @@ -74,13 +64,13 @@ class FakeYDL(YoutubeDL): # Different instances of the downloader can't share the same dictionary # some test set the "sublang" parameter, which would break the md5 checks. params = get_params(override=override) - super(FakeYDL, self).__init__(params, auto_init=False) + super().__init__(params, auto_init=False) self.result = [] - def to_screen(self, s, skip_eol=None): + def to_screen(self, s, *args, **kwargs): print(s) - def trouble(self, s, tb=None): + def trouble(self, s, *args, **kwargs): raise Exception(s) def download(self, x): @@ -90,56 +80,59 @@ class FakeYDL(YoutubeDL): # Silence an expected warning matching a regex old_report_warning = self.report_warning - def report_warning(self, message): + def report_warning(self, message, *args, **kwargs): if re.match(regex, message): return - old_report_warning(message) + old_report_warning(message, *args, **kwargs) self.report_warning = types.MethodType(report_warning, self) def gettestcases(include_onlymatching=False): for ie in hypervideo_dl.extractor.gen_extractors(): - for tc in ie.get_testcases(include_onlymatching): + yield from ie.get_testcases(include_onlymatching) + + +def getwebpagetestcases(): + for ie in hypervideo_dl.extractor.gen_extractors(): + for tc in ie.get_webpage_testcases(): + tc.setdefault('add_ie', []).append('Generic') yield tc -md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest() +md5 = lambda s: hashlib.md5(s.encode()).hexdigest() def expect_value(self, got, expected, field): - if isinstance(expected, compat_str) and expected.startswith('re:'): + if isinstance(expected, str) and expected.startswith('re:'): match_str = expected[len('re:'):] match_rex = re.compile(match_str) self.assertTrue( - isinstance(got, compat_str), - 'Expected a %s object, but got %s for field %s' % ( - compat_str.__name__, type(got).__name__, field)) + isinstance(got, str), + f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') self.assertTrue( match_rex.match(got), - 'field %s (value: %r) should match %r' % (field, got, match_str)) - elif isinstance(expected, compat_str) and expected.startswith('startswith:'): + f'field {field} (value: {got!r}) should match {match_str!r}') + elif isinstance(expected, str) and expected.startswith('startswith:'): start_str = expected[len('startswith:'):] self.assertTrue( - isinstance(got, compat_str), - 'Expected a %s object, but got %s for field %s' % ( - compat_str.__name__, type(got).__name__, field)) + isinstance(got, str), + f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') self.assertTrue( got.startswith(start_str), - 'field %s (value: %r) should start with %r' % (field, got, start_str)) - elif isinstance(expected, compat_str) and expected.startswith('contains:'): + f'field {field} (value: {got!r}) should start with {start_str!r}') + elif isinstance(expected, str) and expected.startswith('contains:'): contains_str = expected[len('contains:'):] self.assertTrue( - isinstance(got, compat_str), - 'Expected a %s object, but got %s for field %s' % ( - compat_str.__name__, type(got).__name__, field)) + isinstance(got, str), + f'Expected a {str.__name__} object, but got {type(got).__name__} for field {field}') self.assertTrue( contains_str in got, - 'field %s (value: %r) should contain %r' % (field, got, contains_str)) + f'field {field} (value: {got!r}) should contain {contains_str!r}') elif isinstance(expected, type): self.assertTrue( isinstance(got, expected), - 'Expected type %r for field %s, but got value %r of type %r' % (expected, field, got, type(got))) + f'Expected type {expected!r} for field {field}, but got value {got!r} of type {type(got)!r}') elif isinstance(expected, dict) and isinstance(got, dict): expect_dict(self, got, expected) elif isinstance(expected, list) and isinstance(got, list): @@ -156,16 +149,15 @@ def expect_value(self, got, expected, field): index, field, type_expected, type_got)) expect_value(self, item_got, item_expected, field) else: - if isinstance(expected, compat_str) and expected.startswith('md5:'): + if isinstance(expected, str) and expected.startswith('md5:'): self.assertTrue( - isinstance(got, compat_str), - 'Expected field %s to be a unicode object, but got value %r of type %r' % (field, got, type(got))) + isinstance(got, str), + f'Expected field {field} to be a unicode object, but got value {got!r} of type {type(got)!r}') got = 'md5:' + md5(got) - elif isinstance(expected, compat_str) and re.match(r'^(?:min|max)?count:\d+', expected): + elif isinstance(expected, str) and re.match(r'^(?:min|max)?count:\d+', expected): self.assertTrue( isinstance(got, (list, dict)), - 'Expected field %s to be a list or a dict, but it is of type %s' % ( - field, type(got).__name__)) + f'Expected field {field} to be a list or a dict, but it is of type {type(got).__name__}') op, _, expected_num = expected.partition(':') expected_num = int(expected_num) if op == 'mincount': @@ -185,7 +177,7 @@ def expect_value(self, got, expected, field): return self.assertEqual( expected, got, - 'Invalid value for field %s, expected %r, got %r' % (field, expected, got)) + f'Invalid value for field {field}, expected {expected!r}, got {got!r}') def expect_dict(self, got_dict, expected_dict): @@ -230,6 +222,10 @@ def sanitize_got_info_dict(got_dict): if test_info_dict.get('display_id') == test_info_dict.get('id'): test_info_dict.pop('display_id') + # Check url for flat entries + if got_dict.get('_type', 'video') != 'video' and got_dict.get('url'): + test_info_dict['url'] = got_dict['url'] + return test_info_dict @@ -243,33 +239,31 @@ def expect_info_dict(self, got_dict, expected_dict): for key in mandatory_fields: self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key) # Check for mandatory fields that are automatically set by YoutubeDL - for key in ['webpage_url', 'extractor', 'extractor_key']: - self.assertTrue(got_dict.get(key), 'Missing field: %s' % key) + if got_dict.get('_type', 'video') == 'video': + for key in ['webpage_url', 'extractor', 'extractor_key']: + self.assertTrue(got_dict.get(key), 'Missing field: %s' % key) test_info_dict = sanitize_got_info_dict(got_dict) missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys()) if missing_keys: def _repr(v): - if isinstance(v, compat_str): + if isinstance(v, str): return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n') elif isinstance(v, type): return v.__name__ else: return repr(v) - info_dict_str = '' - if len(missing_keys) != len(expected_dict): - info_dict_str += ''.join( - ' %s: %s,\n' % (_repr(k), _repr(v)) - for k, v in test_info_dict.items() if k not in missing_keys) - - if info_dict_str: - info_dict_str += '\n' + info_dict_str = ''.join( + f' {_repr(k)}: {_repr(v)},\n' + for k, v in test_info_dict.items() if k not in missing_keys) + if info_dict_str: + info_dict_str += '\n' info_dict_str += ''.join( - ' %s: %s,\n' % (_repr(k), _repr(test_info_dict[k])) + f' {_repr(k)}: {_repr(test_info_dict[k])},\n' for k in missing_keys) - write_string( - '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr) + info_dict_str = '\n\'info_dict\': {\n' + info_dict_str + '},\n' + write_string(info_dict_str.replace('\n', '\n '), out=sys.stderr) self.assertFalse( missing_keys, 'Missing keys in test definition: %s' % ( @@ -295,30 +289,30 @@ def assertRegexpMatches(self, text, regexp, msg=None): def assertGreaterEqual(self, got, expected, msg=None): if not (got >= expected): if msg is None: - msg = '%r not greater than or equal to %r' % (got, expected) + msg = f'{got!r} not greater than or equal to {expected!r}' self.assertTrue(got >= expected, msg) def assertLessEqual(self, got, expected, msg=None): if not (got <= expected): if msg is None: - msg = '%r not less than or equal to %r' % (got, expected) + msg = f'{got!r} not less than or equal to {expected!r}' self.assertTrue(got <= expected, msg) def assertEqual(self, got, expected, msg=None): if not (got == expected): if msg is None: - msg = '%r not equal to %r' % (got, expected) + msg = f'{got!r} not equal to {expected!r}' self.assertTrue(got == expected, msg) def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning - def _report_warning(w): + def _report_warning(w, *args, **kwargs): if not any(re.search(w_re, w) for w_re in warnings_re): - real_warning(w) + real_warning(w, *args, **kwargs) ydl.report_warning = _report_warning diff --git a/test/parameters.json b/test/parameters.json index 06fe3e3..8789ce1 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -44,6 +44,6 @@ "writesubtitles": false, "allsubtitles": false, "listsubtitles": false, - "socket_timeout": 20, - "fixup": "never" + "fixup": "never", + "allow_playlist_files": false } diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 8494105..529da52 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -1,27 +1,32 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - # Allow direct execution -import io import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, expect_dict, expect_value, http_server_port -from hypervideo_dl.compat import compat_etree_fromstring, compat_http_server -from hypervideo_dl.extractor.common import InfoExtractor -from hypervideo_dl.extractor import YoutubeIE, get_info_extractor -from hypervideo_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError + +import http.server import threading +from test.helper import FakeYDL, expect_dict, expect_value, http_server_port +from hypervideo_dl.compat import compat_etree_fromstring +from hypervideo_dl.extractor import YoutubeIE, get_info_extractor +from hypervideo_dl.extractor.common import InfoExtractor +from hypervideo_dl.utils import ( + ExtractorError, + RegexNotFoundError, + encode_data_uri, + strip_jsonp, +) TEAPOT_RESPONSE_STATUS = 418 TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>" -class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): +class InfoExtractorTestRequestHandler(http.server.BaseHTTPRequestHandler): def log_message(self, format, *args): pass @@ -36,7 +41,9 @@ class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler) class DummyIE(InfoExtractor): - pass + def _sort_formats(self, formats, field_preference=[]): + self._downloader.sort_formats( + {'formats': formats, '_format_sort_fields': field_preference}) class TestInfoExtractor(unittest.TestCase): @@ -500,6 +507,24 @@ class TestInfoExtractor(unittest.TestCase): }], }) + # from https://0000.studio/ + # with type attribute but without extension in URL + expect_dict( + self, + self.ie._parse_html5_media_entries( + 'https://0000.studio', + r''' + <video src="https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92" + controls="controls" type="video/mp4" preload="metadata" autoplay="autoplay" playsinline class="object-contain"> + </video> + ''', None)[0], + { + 'formats': [{ + 'url': 'https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92', + 'ext': 'mp4', + }], + }) + def test_extract_jwplayer_data_realworld(self): # from http://www.suffolk.edu/sjc/ expect_dict( @@ -1011,8 +1036,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for m3u8_file, m3u8_url, expected_formats, expected_subs in _TEST_CASES: - with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file, - mode='r', encoding='utf-8') as f: + with open('./test/testdata/m3u8/%s.m3u8' % m3u8_file, encoding='utf-8') as f: formats, subs = self.ie._parse_m3u8_formats_and_subtitles( f.read(), m3u8_url, ext='mp4') self.ie._sort_formats(formats) @@ -1357,10 +1381,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for mpd_file, mpd_url, mpd_base_url, expected_formats, expected_subtitles in _TEST_CASES: - with io.open('./test/testdata/mpd/%s.mpd' % mpd_file, - mode='r', encoding='utf-8') as f: + with open('./test/testdata/mpd/%s.mpd' % mpd_file, encoding='utf-8') as f: formats, subtitles = self.ie._parse_mpd_formats_and_subtitles( - compat_etree_fromstring(f.read().encode('utf-8')), + compat_etree_fromstring(f.read().encode()), mpd_base_url=mpd_base_url, mpd_url=mpd_url) self.ie._sort_formats(formats) expect_value(self, formats, expected_formats, None) @@ -1546,13 +1569,298 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] }, ), + ( + 'ec-3_test', + 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + [{ + 'format_id': 'audio_deu_1-224', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'isma', + 'tbr': 224, + 'asr': 48000, + 'vcodec': 'none', + 'acodec': 'EC-3', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'audio', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 0, + 'height': 0, + 'fourcc': 'EC-3', + 'language': 'deu', + 'codec_private_data': '00063F000000AF87FBA7022DFB42A4D405CD93843BDD0700200F00', + 'sampling_rate': 48000, + 'channels': 6, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'audio_ext': 'isma', + 'video_ext': 'none', + 'abr': 224, + }, { + 'format_id': 'audio_deu-127', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'isma', + 'tbr': 127, + 'asr': 48000, + 'vcodec': 'none', + 'acodec': 'AACL', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'audio', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 0, + 'height': 0, + 'fourcc': 'AACL', + 'language': 'deu', + 'codec_private_data': '1190', + 'sampling_rate': 48000, + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'audio_ext': 'isma', + 'video_ext': 'none', + 'abr': 127, + }, { + 'format_id': 'video_deu-23', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 384, + 'height': 216, + 'tbr': 23, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 384, + 'height': 216, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '000000016742C00CDB06077E5C05A808080A00000300020000030009C0C02EE0177CC6300F142AE00000000168CA8DC8', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 23, + }, { + 'format_id': 'video_deu-403', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 400, + 'height': 224, + 'tbr': 403, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 400, + 'height': 224, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D4014E98323B602D4040405000003000100000300320F1429380000000168EAECF2', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 403, + }, { + 'format_id': 'video_deu-680', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 640, + 'height': 360, + 'tbr': 680, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 640, + 'height': 360, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D401EE981405FF2E02D4040405000000300100000030320F162D3800000000168EAECF2', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 680, + }, { + 'format_id': 'video_deu-1253', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 640, + 'height': 360, + 'tbr': 1253, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 640, + 'height': 360, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D401EE981405FF2E02D4040405000000300100000030320F162D3800000000168EAECF2', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 1253, + }, { + 'format_id': 'video_deu-2121', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 768, + 'height': 432, + 'tbr': 2121, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 768, + 'height': 432, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D401EECA0601BD80B50101014000003000400000300C83C58B6580000000168E93B3C80', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 2121, + }, { + 'format_id': 'video_deu-3275', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 1280, + 'height': 720, + 'tbr': 3275, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 1280, + 'height': 720, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D4020ECA02802DD80B501010140000003004000000C83C60C65800000000168E93B3C80', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 3275, + }, { + 'format_id': 'video_deu-5300', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 1920, + 'height': 1080, + 'tbr': 5300, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 1920, + 'height': 1080, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D4028ECA03C0113F2E02D4040405000000300100000030320F18319600000000168E93B3C80', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 5300, + }, { + 'format_id': 'video_deu-8079', + 'url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'manifest_url': 'https://smstr01.dmm.t-online.de/smooth24/smoothstream_m1/streaming/sony/9221438342941275747/636887760842957027/25_km_h-Trailer-9221571562372022953_deu_20_1300k_HD_H_264_ISMV.ism/Manifest', + 'ext': 'ismv', + 'width': 1920, + 'height': 1080, + 'tbr': 8079, + 'vcodec': 'AVC1', + 'acodec': 'none', + 'protocol': 'ism', + '_download_params': + { + 'stream_type': 'video', + 'duration': 370000000, + 'timescale': 10000000, + 'width': 1920, + 'height': 1080, + 'fourcc': 'AVC1', + 'language': 'deu', + 'codec_private_data': '00000001674D4028ECA03C0113F2E02D4040405000000300100000030320F18319600000000168E93B3C80', + 'channels': 2, + 'bits_per_sample': 16, + 'nal_unit_length_field': 4 + }, + 'video_ext': 'ismv', + 'audio_ext': 'none', + 'vbr': 8079, + }], + {}, + ), ] for ism_file, ism_url, expected_formats, expected_subtitles in _TEST_CASES: - with io.open('./test/testdata/ism/%s.Manifest' % ism_file, - mode='r', encoding='utf-8') as f: + with open('./test/testdata/ism/%s.Manifest' % ism_file, encoding='utf-8') as f: formats, subtitles = self.ie._parse_ism_formats_and_subtitles( - compat_etree_fromstring(f.read().encode('utf-8')), ism_url=ism_url) + compat_etree_fromstring(f.read().encode()), ism_url=ism_url) self.ie._sort_formats(formats) expect_value(self, formats, expected_formats, None) expect_value(self, subtitles, expected_subtitles, None) @@ -1576,10 +1884,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for f4m_file, f4m_url, expected_formats in _TEST_CASES: - with io.open('./test/testdata/f4m/%s.f4m' % f4m_file, - mode='r', encoding='utf-8') as f: + with open('./test/testdata/f4m/%s.f4m' % f4m_file, encoding='utf-8') as f: formats = self.ie._parse_f4m_formats( - compat_etree_fromstring(f.read().encode('utf-8')), + compat_etree_fromstring(f.read().encode()), f4m_url, None) self.ie._sort_formats(formats) expect_value(self, formats, expected_formats, None) @@ -1624,10 +1931,9 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for xspf_file, xspf_url, expected_entries in _TEST_CASES: - with io.open('./test/testdata/xspf/%s.xspf' % xspf_file, - mode='r', encoding='utf-8') as f: + with open('./test/testdata/xspf/%s.xspf' % xspf_file, encoding='utf-8') as f: entries = self.ie._parse_xspf( - compat_etree_fromstring(f.read().encode('utf-8')), + compat_etree_fromstring(f.read().encode()), xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url) expect_value(self, entries, expected_entries, None) for i in range(len(entries)): @@ -1640,7 +1946,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ # or the underlying `_download_webpage_handle` returning no content # when a response matches `expected_status`. - httpd = compat_http_server.HTTPServer( + httpd = http.server.HTTPServer( ('127.0.0.1', 0), InfoExtractorTestRequestHandler) port = http_server_port(httpd) server_thread = threading.Thread(target=httpd.serve_forever) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index fe0fd35..2d4e827 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -1,38 +1,44 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import copy import json +import urllib.error from test.helper import FakeYDL, assertRegexpMatches from hypervideo_dl import YoutubeDL -from hypervideo_dl.compat import compat_os_name, compat_setenv, compat_str, compat_urllib_error +from hypervideo_dl.compat import compat_os_name from hypervideo_dl.extractor import YoutubeIE from hypervideo_dl.extractor.common import InfoExtractor from hypervideo_dl.postprocessor.common import PostProcessor -from hypervideo_dl.utils import ExtractorError, int_or_none, match_filter_func, LazyList +from hypervideo_dl.utils import ( + ExtractorError, + LazyList, + OnDemandPagedList, + int_or_none, + match_filter_func, +) TEST_URL = 'http://localhost/sample.mp4' class YDL(FakeYDL): def __init__(self, *args, **kwargs): - super(YDL, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.downloaded_info_dicts = [] self.msgs = [] def process_info(self, info_dict): self.downloaded_info_dicts.append(info_dict.copy()) - def to_screen(self, msg): + def to_screen(self, msg, *args, **kwargs): self.msgs.append(msg) def dl(self, *args, **kwargs): @@ -62,8 +68,7 @@ class TestFormatSelection(unittest.TestCase): {'ext': 'mp4', 'height': 460, 'url': TEST_URL}, ] info_dict = _make_result(formats) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['ext'], 'webm') @@ -76,8 +81,7 @@ class TestFormatSelection(unittest.TestCase): {'ext': 'mp4', 'height': 1080, 'url': TEST_URL}, ] info_dict['formats'] = formats - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['ext'], 'mp4') @@ -91,8 +95,7 @@ class TestFormatSelection(unittest.TestCase): {'ext': 'flv', 'height': 720, 'url': TEST_URL}, ] info_dict['formats'] = formats - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['ext'], 'mp4') @@ -104,15 +107,14 @@ class TestFormatSelection(unittest.TestCase): {'ext': 'webm', 'height': 720, 'url': TEST_URL}, ] info_dict['formats'] = formats - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['ext'], 'webm') def test_format_selection(self): formats = [ - {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL}, + {'format_id': '35', 'ext': 'mp4', 'preference': 0, 'url': TEST_URL}, {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL}, {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL}, {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL}, @@ -180,22 +182,19 @@ class TestFormatSelection(unittest.TestCase): info_dict = _make_result(formats) ydl = YDL({'format': 'best'}) - ie = YoutubeIE(ydl) - ie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(copy.deepcopy(info_dict)) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], 'aac-64') ydl = YDL({'format': 'mp3'}) - ie = YoutubeIE(ydl) - ie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(copy.deepcopy(info_dict)) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], 'mp3-64') ydl = YDL({'prefer_free_formats': True}) - ie = YoutubeIE(ydl) - ie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(copy.deepcopy(info_dict)) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], 'ogg-64') @@ -340,8 +339,7 @@ class TestFormatSelection(unittest.TestCase): info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': 'bestvideo+bestaudio'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], '248+172') @@ -349,40 +347,35 @@ class TestFormatSelection(unittest.TestCase): info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], '38') info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': 'bestvideo/best,bestaudio'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] self.assertEqual(downloaded_ids, ['137', '141']) info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] self.assertEqual(downloaded_ids, ['137+141', '248+141']) info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] self.assertEqual(downloaded_ids, ['136+141', '247+141']) info_dict = _make_result(list(formats_order), extractor='youtube') ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts] self.assertEqual(downloaded_ids, ['248+141']) @@ -390,16 +383,14 @@ class TestFormatSelection(unittest.TestCase): for f1, f2 in zip(formats_order, formats_order[1:]): info_dict = _make_result([f1, f2], extractor='youtube') ydl = YDL({'format': 'best/bestvideo'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], f1['format_id']) info_dict = _make_result([f2, f1], extractor='youtube') ydl = YDL({'format': 'best/bestvideo'}) - yie = YoutubeIE(ydl) - yie._sort_formats(info_dict['formats']) + ydl.sort_formats(info_dict) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], f1['format_id']) @@ -474,7 +465,7 @@ class TestFormatSelection(unittest.TestCase): for f in formats: f['url'] = 'http://_/' f['ext'] = 'unknown' - info_dict = _make_result(formats) + info_dict = _make_result(formats, _format_sort_fields=('id', )) ydl = YDL({'format': 'best[filesize<3000]'}) ydl.process_ie_result(info_dict) @@ -551,11 +542,11 @@ class TestYoutubeDL(unittest.TestCase): def s_formats(lang, autocaption=False): return [{ 'ext': ext, - 'url': 'http://localhost/video.%s.%s' % (lang, ext), + 'url': f'http://localhost/video.{lang}.{ext}', '_auto': autocaption, } for ext in ['vtt', 'srt', 'ass']] - subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es']) - auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es']) + subtitles = {l: s_formats(l) for l in ['en', 'fr', 'es']} + auto_captions = {l: s_formats(l, True) for l in ['it', 'pt', 'es']} info_dict = { 'id': 'test', 'title': 'Test', @@ -580,7 +571,7 @@ class TestYoutubeDL(unittest.TestCase): result = get_info({'writesubtitles': True}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['en'])) + self.assertEqual(set(subs.keys()), {'en'}) self.assertTrue(subs['en'].get('data') is None) self.assertEqual(subs['en']['ext'], 'ass') @@ -591,39 +582,39 @@ class TestYoutubeDL(unittest.TestCase): result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['es', 'fr'])) + self.assertEqual(set(subs.keys()), {'es', 'fr'}) result = get_info({'writesubtitles': True, 'subtitleslangs': ['all', '-en']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['es', 'fr'])) + self.assertEqual(set(subs.keys()), {'es', 'fr'}) result = get_info({'writesubtitles': True, 'subtitleslangs': ['en', 'fr', '-en']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['fr'])) + self.assertEqual(set(subs.keys()), {'fr'}) result = get_info({'writesubtitles': True, 'subtitleslangs': ['-en', 'en']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['en'])) + self.assertEqual(set(subs.keys()), {'en'}) result = get_info({'writesubtitles': True, 'subtitleslangs': ['e.+']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['es', 'en'])) + self.assertEqual(set(subs.keys()), {'es', 'en'}) result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['es', 'pt'])) + self.assertEqual(set(subs.keys()), {'es', 'pt'}) self.assertFalse(subs['es']['_auto']) self.assertTrue(subs['pt']['_auto']) result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']}) subs = result['requested_subtitles'] self.assertTrue(subs) - self.assertEqual(set(subs.keys()), set(['es', 'pt'])) + self.assertEqual(set(subs.keys()), {'es', 'pt'}) self.assertTrue(subs['es']['_auto']) self.assertTrue(subs['pt']['_auto']) @@ -654,15 +645,19 @@ class TestYoutubeDL(unittest.TestCase): 'duration': 100000, 'playlist_index': 1, 'playlist_autonumber': 2, - '_last_playlist_index': 100, + '__last_playlist_index': 100, 'n_entries': 10, - 'formats': [{'id': 'id 1'}, {'id': 'id 2'}, {'id': 'id 3'}] + 'formats': [ + {'id': 'id 1', 'height': 1080, 'width': 1920}, + {'id': 'id 2', 'height': 720}, + {'id': 'id 3'} + ] } def test_prepare_outtmpl_and_filename(self): def test(tmpl, expected, *, info=None, **params): params['outtmpl'] = tmpl - ydl = YoutubeDL(params) + ydl = FakeYDL(params) ydl._num_downloads = 1 self.assertEqual(ydl.validate_outtmpl(tmpl), None) @@ -716,13 +711,14 @@ class TestYoutubeDL(unittest.TestCase): test('%(id)s', '-abcd', info={'id': '-abcd'}) test('%(id)s', '.abcd', info={'id': '.abcd'}) test('%(id)s', 'ab__cd', info={'id': 'ab__cd'}) - test('%(id)s', ('ab:cd', 'ab -cd'), info={'id': 'ab:cd'}) + test('%(id)s', ('ab:cd', 'ab：cd'), info={'id': 'ab:cd'}) test('%(id.0)s', '-', info={'id': '--'}) # Invalid templates self.assertTrue(isinstance(YoutubeDL.validate_outtmpl('%(title)'), ValueError)) test('%(invalid@tmpl|def)s', 'none', outtmpl_na_placeholder='none') test('%(..)s', 'NA') + test('%(formats.{id)s', 'NA') # Entire info_dict def expect_same_infodict(out): @@ -764,7 +760,7 @@ class TestYoutubeDL(unittest.TestCase): test('a%(width|)d', 'a', outtmpl_na_placeholder='none') FORMATS = self.outtmpl_info['formats'] - sanitize = lambda x: x.replace(':', ' -').replace('"', "'").replace('\n', ' ') + sanitize = lambda x: x.replace(':', '：').replace('"', "＂").replace('\n', ' ') # Custom type casting test('%(formats.:.id)l', 'id 1, id 2, id 3') @@ -782,13 +778,13 @@ class TestYoutubeDL(unittest.TestCase): test('%(filesize)#D', '1Ki') test('%(height)5.2D', ' 1.08k') test('%(title4)#S', 'foo_bar_test') - test('%(title4).10S', ('foo \'bar\' ', 'foo \'bar\'' + ('#' if compat_os_name == 'nt' else ' '))) + test('%(title4).10S', ('foo ＂bar＂ ', 'foo ＂bar＂' + ('#' if compat_os_name == 'nt' else ' '))) if compat_os_name == 'nt': - test('%(title4)q', ('"foo \\"bar\\" test"', "'foo _'bar_' test'")) - test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', "'id 1' 'id 2' 'id 3'")) - test('%(formats.0.id)#q', ('"id 1"', "'id 1'")) + test('%(title4)q', ('"foo \\"bar\\" test"', "＂foo ⧹＂bar⧹＂ test＂")) + test('%(formats.:.id)#q', ('"id 1" "id 2" "id 3"', '＂id 1＂ ＂id 2＂ ＂id 3＂')) + test('%(formats.0.id)#q', ('"id 1"', '＂id 1＂')) else: - test('%(title4)q', ('\'foo "bar" test\'', "'foo 'bar' test'")) + test('%(title4)q', ('\'foo "bar" test\'', '\'foo ＂bar＂ test\'')) test('%(formats.:.id)#q', "'id 1' 'id 2' 'id 3'") test('%(formats.0.id)#q', "'id 1'") @@ -807,6 +803,12 @@ class TestYoutubeDL(unittest.TestCase): test('%(formats.:2:-1)r', repr(FORMATS[:2:-1])) test('%(formats.0.id.-1+id)f', '1235.000000') test('%(formats.0.id.-1+formats.1.id.-1)d', '3') + out = json.dumps([{'id': f['id'], 'height.:2': str(f['height'])[:2]} + if 'height' in f else {'id': f['id']} + for f in FORMATS]) + test('%(formats.:.{id,height.:2})j', (out, sanitize(out))) + test('%(formats.:.{id,height}.id)l', ', '.join(f['id'] for f in FORMATS)) + test('%(.{id,title})j', ('{"id": "1234"}', '{＂id＂： ＂1234＂}')) # Alternates test('%(title,id)s', '1234') @@ -833,21 +835,21 @@ class TestYoutubeDL(unittest.TestCase): # test('%(foo|)s', ('', '_')) # fixme # Environment variable expansion for prepare_filename - compat_setenv('__hypervideo_dl_var', 'expanded') + os.environ['__hypervideo_dl_var'] = 'expanded' envvar = '%__hypervideo_dl_var%' if compat_os_name == 'nt' else '$__hypervideo_dl_var' test(envvar, (envvar, 'expanded')) if compat_os_name == 'nt': test('%s%', ('%s%', '%s%')) - compat_setenv('s', 'expanded') + os.environ['s'] = 'expanded' test('%s%', ('%s%', 'expanded')) # %s% should be expanded before escaping %s - compat_setenv('(test)s', 'expanded') + os.environ['(test)s'] = 'expanded' test('%(test)s%', ('NA%', 'expanded')) # Environment should take priority over template # Path expansion and escaping test('Hello %(title1)s', 'Hello $PATH') test('Hello %(title2)s', 'Hello %PATH%') - test('%(title3)s', ('foo/bar\\test', 'foo_bar_test')) - test('folder/%(title3)s', ('folder/foo/bar\\test', 'folder%sfoo_bar_test' % os.path.sep)) + test('%(title3)s', ('foo/bar\\test', 'foo⧸bar⧹test')) + test('folder/%(title3)s', ('folder/foo/bar\\test', 'folder%sfoo⧸bar⧹test' % os.path.sep)) def test_format_note(self): ydl = YoutubeDL() @@ -982,41 +984,80 @@ class TestYoutubeDL(unittest.TestCase): self.assertEqual(res, []) def test_playlist_items_selection(self): - entries = [{ - 'id': compat_str(i), - 'title': compat_str(i), - 'url': TEST_URL, - } for i in range(1, 5)] - playlist = { - '_type': 'playlist', - 'id': 'test', - 'entries': entries, - 'extractor': 'test:playlist', - 'extractor_key': 'test:playlist', - 'webpage_url': 'http://example.com', - } + INDICES, PAGE_SIZE = list(range(1, 11)), 3 - def get_downloaded_info_dicts(params): + def entry(i, evaluated): + evaluated.append(i) + return { + 'id': str(i), + 'title': str(i), + 'url': TEST_URL, + } + + def pagedlist_entries(evaluated): + def page_func(n): + start = PAGE_SIZE * n + for i in INDICES[start: start + PAGE_SIZE]: + yield entry(i, evaluated) + return OnDemandPagedList(page_func, PAGE_SIZE) + + def page_num(i): + return (i + PAGE_SIZE - 1) // PAGE_SIZE + + def generator_entries(evaluated): + for i in INDICES: + yield entry(i, evaluated) + + def list_entries(evaluated): + return list(generator_entries(evaluated)) + + def lazylist_entries(evaluated): + return LazyList(generator_entries(evaluated)) + + def get_downloaded_info_dicts(params, entries): ydl = YDL(params) - # make a deep copy because the dictionary and nested entries - # can be modified - ydl.process_ie_result(copy.deepcopy(playlist)) + ydl.process_ie_result({ + '_type': 'playlist', + 'id': 'test', + 'extractor': 'test:playlist', + 'extractor_key': 'test:playlist', + 'webpage_url': 'http://example.com', + 'entries': entries, + }) return ydl.downloaded_info_dicts - def test_selection(params, expected_ids): - results = [ - (v['playlist_autonumber'] - 1, (int(v['id']), v['playlist_index'])) - for v in get_downloaded_info_dicts(params)] - self.assertEqual(results, list(enumerate(zip(expected_ids, expected_ids)))) - - test_selection({}, [1, 2, 3, 4]) - test_selection({'playlistend': 10}, [1, 2, 3, 4]) - test_selection({'playlistend': 2}, [1, 2]) - test_selection({'playliststart': 10}, []) - test_selection({'playliststart': 2}, [2, 3, 4]) - test_selection({'playlist_items': '2-4'}, [2, 3, 4]) + def test_selection(params, expected_ids, evaluate_all=False): + expected_ids = list(expected_ids) + if evaluate_all: + generator_eval = pagedlist_eval = INDICES + elif not expected_ids: + generator_eval = pagedlist_eval = [] + else: + generator_eval = INDICES[0: max(expected_ids)] + pagedlist_eval = INDICES[PAGE_SIZE * page_num(min(expected_ids)) - PAGE_SIZE: + PAGE_SIZE * page_num(max(expected_ids))] + + for name, func, expected_eval in ( + ('list', list_entries, INDICES), + ('Generator', generator_entries, generator_eval), + # ('LazyList', lazylist_entries, generator_eval), # Generator and LazyList follow the exact same code path + ('PagedList', pagedlist_entries, pagedlist_eval), + ): + evaluated = [] + entries = func(evaluated) + results = [(v['playlist_autonumber'] - 1, (int(v['id']), v['playlist_index'])) + for v in get_downloaded_info_dicts(params, entries)] + self.assertEqual(results, list(enumerate(zip(expected_ids, expected_ids))), f'Entries of {name} for {params}') + self.assertEqual(sorted(evaluated), expected_eval, f'Evaluation of {name} for {params}') + + test_selection({}, INDICES) + test_selection({'playlistend': 20}, INDICES, True) + test_selection({'playlistend': 2}, INDICES[:2]) + test_selection({'playliststart': 11}, [], True) + test_selection({'playliststart': 2}, INDICES[1:]) + test_selection({'playlist_items': '2-4'}, INDICES[1:4]) test_selection({'playlist_items': '2,4'}, [2, 4]) - test_selection({'playlist_items': '10'}, []) + test_selection({'playlist_items': '20'}, [], True) test_selection({'playlist_items': '0'}, []) # Tests for https://github.com/ytdl-org/youtube-dl/issues/10591 @@ -1025,15 +1066,37 @@ class TestYoutubeDL(unittest.TestCase): # Tests for https://github.com/hypervideo/hypervideo/issues/720 # https://github.com/hypervideo/hypervideo/issues/302 - test_selection({'playlistreverse': True}, [4, 3, 2, 1]) - test_selection({'playliststart': 2, 'playlistreverse': True}, [4, 3, 2]) + test_selection({'playlistreverse': True}, INDICES[::-1]) + test_selection({'playliststart': 2, 'playlistreverse': True}, INDICES[:0:-1]) test_selection({'playlist_items': '2,4', 'playlistreverse': True}, [4, 2]) test_selection({'playlist_items': '4,2'}, [4, 2]) + # Tests for --playlist-items start:end:step + test_selection({'playlist_items': ':'}, INDICES, True) + test_selection({'playlist_items': '::1'}, INDICES, True) + test_selection({'playlist_items': '::-1'}, INDICES[::-1], True) + test_selection({'playlist_items': ':6'}, INDICES[:6]) + test_selection({'playlist_items': ':-6'}, INDICES[:-5], True) + test_selection({'playlist_items': '-1:6:-2'}, INDICES[:4:-2], True) + test_selection({'playlist_items': '9:-6:-2'}, INDICES[8:3:-2], True) + + test_selection({'playlist_items': '1:inf:2'}, INDICES[::2], True) + test_selection({'playlist_items': '-2:inf'}, INDICES[-2:], True) + test_selection({'playlist_items': ':inf:-1'}, [], True) + test_selection({'playlist_items': '0-2:2'}, [2]) + test_selection({'playlist_items': '1-:2'}, INDICES[::2], True) + test_selection({'playlist_items': '0--2:2'}, INDICES[1:-1:2], True) + + test_selection({'playlist_items': '10::3'}, [10], True) + test_selection({'playlist_items': '-1::3'}, [10], True) + test_selection({'playlist_items': '11::3'}, [], True) + test_selection({'playlist_items': '-15::2'}, INDICES[1::2], True) + test_selection({'playlist_items': '-15::15'}, [], True) + def test_urlopen_no_file_protocol(self): # see https://github.com/ytdl-org/youtube-dl/issues/8227 ydl = YDL() - self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd') + self.assertRaises(urllib.error.URLError, ydl.urlopen, 'file:///etc/passwd') def test_do_not_override_ie_key_in_url_transparent(self): ydl = YDL() @@ -1082,7 +1145,7 @@ class TestYoutubeDL(unittest.TestCase): class _YDL(YDL): def __init__(self, *args, **kwargs): - super(_YDL, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def trouble(self, s, tb=None): pass @@ -1119,7 +1182,7 @@ class TestYoutubeDL(unittest.TestCase): def _entries(self): for n in range(3): - video_id = compat_str(n) + video_id = str(n) yield { '_type': 'url_transparent', 'ie_key': VideoIE.ie_key(), diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py index 2ce0070..26922d6 100644 --- a/test/test_YoutubeDLCookieJar.py +++ b/test/test_YoutubeDLCookieJar.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals +# Allow direct execution import os -import re import sys -import tempfile import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import re +import tempfile + from hypervideo_dl.utils import YoutubeDLCookieJar @@ -20,7 +21,7 @@ class TestYoutubeDLCookieJar(unittest.TestCase): tf = tempfile.NamedTemporaryFile(delete=False) try: cookiejar.save(filename=tf.name, ignore_discard=True, ignore_expires=True) - temp = tf.read().decode('utf-8') + temp = tf.read().decode() self.assertTrue(re.search( r'www\.foobar\.foobar\s+FALSE\s+/\s+TRUE\s+0\s+YoutubeDLExpiresEmpty\s+YoutubeDLExpiresEmptyValue', temp)) self.assertTrue(re.search( diff --git a/test/test_aes.py b/test/test_aes.py index 9d260b5..0f35bc2 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -1,30 +1,33 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import base64 + from hypervideo_dl.aes import ( - aes_decrypt, - aes_encrypt, - aes_ecb_encrypt, - aes_ecb_decrypt, aes_cbc_decrypt, aes_cbc_decrypt_bytes, aes_cbc_encrypt, aes_ctr_decrypt, aes_ctr_encrypt, + aes_decrypt, + aes_decrypt_text, + aes_ecb_decrypt, + aes_ecb_encrypt, + aes_encrypt, aes_gcm_decrypt_and_verify, aes_gcm_decrypt_and_verify_bytes, - aes_decrypt_text, - BLOCK_SIZE_BYTES, + key_expansion, + pad_block, ) -from hypervideo_dl.compat import compat_pycrypto_AES +from hypervideo_dl.dependencies import Cryptodome_AES from hypervideo_dl.utils import bytes_to_intlist, intlist_to_bytes -import base64 # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py' @@ -45,7 +48,7 @@ class TestAES(unittest.TestCase): data = b'\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6\x27\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd' decrypted = intlist_to_bytes(aes_cbc_decrypt(bytes_to_intlist(data), self.key, self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if compat_pycrypto_AES: + if Cryptodome_AES: decrypted = aes_cbc_decrypt_bytes(data, intlist_to_bytes(self.key), intlist_to_bytes(self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) @@ -75,32 +78,31 @@ class TestAES(unittest.TestCase): decrypted = intlist_to_bytes(aes_gcm_decrypt_and_verify( bytes_to_intlist(data), self.key, bytes_to_intlist(authentication_tag), self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) - if compat_pycrypto_AES: + if Cryptodome_AES: decrypted = aes_gcm_decrypt_and_verify_bytes( data, intlist_to_bytes(self.key), authentication_tag, intlist_to_bytes(self.iv[:12])) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) def test_decrypt_text(self): - password = intlist_to_bytes(self.key).decode('utf-8') + password = intlist_to_bytes(self.key).decode() encrypted = base64.b64encode( intlist_to_bytes(self.iv[:8]) + b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae' - ).decode('utf-8') + ).decode() decrypted = (aes_decrypt_text(encrypted, password, 16)) self.assertEqual(decrypted, self.secret_msg) - password = intlist_to_bytes(self.key).decode('utf-8') + password = intlist_to_bytes(self.key).decode() encrypted = base64.b64encode( intlist_to_bytes(self.iv[:8]) + b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83' - ).decode('utf-8') + ).decode() decrypted = (aes_decrypt_text(encrypted, password, 32)) self.assertEqual(decrypted, self.secret_msg) def test_ecb_encrypt(self): data = bytes_to_intlist(self.secret_msg) - data += [0x08] * (BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES) - encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key, self.iv)) + encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key)) self.assertEqual( encrypted, b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:') @@ -110,6 +112,41 @@ class TestAES(unittest.TestCase): decrypted = intlist_to_bytes(aes_ecb_decrypt(data, self.key, self.iv)) self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg) + def test_key_expansion(self): + key = '4f6bdaa39e2f8cb07f5e722d9edef314' + + self.assertEqual(key_expansion(bytes_to_intlist(bytearray.fromhex(key))), [ + 0x4F, 0x6B, 0xDA, 0xA3, 0x9E, 0x2F, 0x8C, 0xB0, 0x7F, 0x5E, 0x72, 0x2D, 0x9E, 0xDE, 0xF3, 0x14, + 0x53, 0x66, 0x20, 0xA8, 0xCD, 0x49, 0xAC, 0x18, 0xB2, 0x17, 0xDE, 0x35, 0x2C, 0xC9, 0x2D, 0x21, + 0x8C, 0xBE, 0xDD, 0xD9, 0x41, 0xF7, 0x71, 0xC1, 0xF3, 0xE0, 0xAF, 0xF4, 0xDF, 0x29, 0x82, 0xD5, + 0x2D, 0xAD, 0xDE, 0x47, 0x6C, 0x5A, 0xAF, 0x86, 0x9F, 0xBA, 0x00, 0x72, 0x40, 0x93, 0x82, 0xA7, + 0xF9, 0xBE, 0x82, 0x4E, 0x95, 0xE4, 0x2D, 0xC8, 0x0A, 0x5E, 0x2D, 0xBA, 0x4A, 0xCD, 0xAF, 0x1D, + 0x54, 0xC7, 0x26, 0x98, 0xC1, 0x23, 0x0B, 0x50, 0xCB, 0x7D, 0x26, 0xEA, 0x81, 0xB0, 0x89, 0xF7, + 0x93, 0x60, 0x4E, 0x94, 0x52, 0x43, 0x45, 0xC4, 0x99, 0x3E, 0x63, 0x2E, 0x18, 0x8E, 0xEA, 0xD9, + 0xCA, 0xE7, 0x7B, 0x39, 0x98, 0xA4, 0x3E, 0xFD, 0x01, 0x9A, 0x5D, 0xD3, 0x19, 0x14, 0xB7, 0x0A, + 0xB0, 0x4E, 0x1C, 0xED, 0x28, 0xEA, 0x22, 0x10, 0x29, 0x70, 0x7F, 0xC3, 0x30, 0x64, 0xC8, 0xC9, + 0xE8, 0xA6, 0xC1, 0xE9, 0xC0, 0x4C, 0xE3, 0xF9, 0xE9, 0x3C, 0x9C, 0x3A, 0xD9, 0x58, 0x54, 0xF3, + 0xB4, 0x86, 0xCC, 0xDC, 0x74, 0xCA, 0x2F, 0x25, 0x9D, 0xF6, 0xB3, 0x1F, 0x44, 0xAE, 0xE7, 0xEC]) + + def test_pad_block(self): + block = [0x21, 0xA0, 0x43, 0xFF] + + self.assertEqual(pad_block(block, 'pkcs7'), + block + [0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C]) + + self.assertEqual(pad_block(block, 'iso7816'), + block + [0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + + self.assertEqual(pad_block(block, 'whitespace'), + block + [0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20]) + + self.assertEqual(pad_block(block, 'zero'), + block + [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + + block = list(range(16)) + for mode in ('pkcs7', 'iso7816', 'whitespace', 'zero'): + self.assertEqual(pad_block(block, mode), block, mode) + if __name__ == '__main__': unittest.main() diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index 9b490d0..034359b 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import try_rm, is_download_test +from test.helper import is_download_test, try_rm from hypervideo_dl import YoutubeDL diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 74634cb..49653f1 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -1,22 +1,17 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - # Allow direct execution import os import sys import unittest -import collections + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import gettestcases +import collections -from hypervideo_dl.extractor import ( - FacebookIE, - gen_extractors, - YoutubeIE, -) +from test.helper import gettestcases +from hypervideo_dl.extractor import FacebookIE, YoutubeIE, gen_extractors class TestAllURLsMatching(unittest.TestCase): @@ -81,11 +76,11 @@ class TestAllURLsMatching(unittest.TestCase): url = tc['url'] for ie in ies: if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): - self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url)) + self.assertTrue(ie.suitable(url), f'{type(ie).__name__} should match URL {url!r}') else: self.assertFalse( ie.suitable(url), - '%s should not match URL %r . That URL belongs to %s.' % (type(ie).__name__, url, tc['name'])) + f'{type(ie).__name__} should not match URL {url!r} . That URL belongs to {tc["name"]}.') def test_keywords(self): self.assertMatch(':ytsubs', ['youtube:subscriptions']) @@ -120,7 +115,7 @@ class TestAllURLsMatching(unittest.TestCase): for (ie_name, ie_list) in name_accu.items(): self.assertEqual( len(ie_list), 1, - 'Multiple extractors with the same IE_NAME "%s" (%s)' % (ie_name, ', '.join(ie_list))) + f'Multiple extractors with the same IE_NAME "{ie_name}" ({", ".join(ie_list)})') if __name__ == '__main__': diff --git a/test/test_cache.py b/test/test_cache.py index 0776e92..f366a15 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,17 +1,15 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals - -import shutil # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import shutil + from test.helper import FakeYDL from hypervideo_dl.cache import Cache diff --git a/test/test_compat.py b/test/test_compat.py index 5f5d354..7a191c0 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -1,80 +1,44 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import struct +import urllib.parse + +from hypervideo_dl import compat from hypervideo_dl.compat import ( - compat_getenv, - compat_setenv, - compat_etree_Element, compat_etree_fromstring, compat_expanduser, - compat_shlex_split, - compat_str, - compat_struct_unpack, - compat_urllib_parse_quote, - compat_urllib_parse_quote_plus, compat_urllib_parse_unquote, - compat_urllib_parse_unquote_plus, compat_urllib_parse_urlencode, ) class TestCompat(unittest.TestCase): - def test_compat_getenv(self): - test_str = 'тест' - compat_setenv('hypervideo_dl_COMPAT_GETENV', test_str) - self.assertEqual(compat_getenv('hypervideo_dl_COMPAT_GETENV'), test_str) - - def test_compat_setenv(self): - test_var = 'hypervideo_dl_COMPAT_SETENV' - test_str = 'тест' - compat_setenv(test_var, test_str) - compat_getenv(test_var) - self.assertEqual(compat_getenv(test_var), test_str) + def test_compat_passthrough(self): + with self.assertWarns(DeprecationWarning): + compat.compat_basestring + + with self.assertWarns(DeprecationWarning): + compat.WINDOWS_VT_MODE + + # TODO: Test submodule + # compat.asyncio.events # Must not raise error def test_compat_expanduser(self): old_home = os.environ.get('HOME') - test_str = r'C:\Documents and Settings\тест\Application Data' - compat_setenv('HOME', test_str) - self.assertEqual(compat_expanduser('~'), test_str) - compat_setenv('HOME', old_home or '') - - def test_all_present(self): - import hypervideo_dl.compat - all_names = hypervideo_dl.compat.__all__ - present_names = set(filter( - lambda c: '_' in c and not c.startswith('_'), - dir(hypervideo_dl.compat))) - set(['unicode_literals']) - self.assertEqual(all_names, sorted(present_names)) - - def test_compat_urllib_parse_quote(self): - self.assertEqual(compat_urllib_parse_quote('abc def'), 'abc%20def') - self.assertEqual(compat_urllib_parse_quote('/user/abc+def'), '/user/abc%2Bdef') - self.assertEqual(compat_urllib_parse_quote('/user/abc+def', safe='+'), '%2Fuser%2Fabc+def') - self.assertEqual(compat_urllib_parse_quote(''), '') - self.assertEqual(compat_urllib_parse_quote('%'), '%25') - self.assertEqual(compat_urllib_parse_quote('%', safe='%'), '%') - self.assertEqual(compat_urllib_parse_quote('津波'), '%E6%B4%A5%E6%B3%A2') - self.assertEqual( - compat_urllib_parse_quote('''<meta property="og:description" content="▁▂▃▄%▅▆▇█" /> -%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''', safe='<>=":%/ \r\n'), - '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%%E2%96%85%E2%96%86%E2%96%87%E2%96%88" /> -%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a''') - self.assertEqual( - compat_urllib_parse_quote('''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%25Things%''', safe='% '), - '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%''') - - def test_compat_urllib_parse_quote_plus(self): - self.assertEqual(compat_urllib_parse_quote_plus('abc def'), 'abc+def') - self.assertEqual(compat_urllib_parse_quote_plus('/abc def'), '%2Fabc+def') + test_str = R'C:\Documents and Settings\тест\Application Data' + try: + os.environ['HOME'] = test_str + self.assertEqual(compat_expanduser('~'), test_str) + finally: + os.environ['HOME'] = old_home or '' def test_compat_urllib_parse_unquote(self): self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def') @@ -96,8 +60,8 @@ class TestCompat(unittest.TestCase): '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''') def test_compat_urllib_parse_unquote_plus(self): - self.assertEqual(compat_urllib_parse_unquote_plus('abc%20def'), 'abc def') - self.assertEqual(compat_urllib_parse_unquote_plus('%7e/abc+def'), '~/abc def') + self.assertEqual(urllib.parse.unquote_plus('abc%20def'), 'abc def') + self.assertEqual(urllib.parse.unquote_plus('%7e/abc+def'), '~/abc def') def test_compat_urllib_parse_urlencode(self): self.assertEqual(compat_urllib_parse_urlencode({'abc': 'def'}), 'abc=def') @@ -109,17 +73,6 @@ class TestCompat(unittest.TestCase): self.assertEqual(compat_urllib_parse_urlencode([(b'abc', 'def')]), 'abc=def') self.assertEqual(compat_urllib_parse_urlencode([(b'abc', b'def')]), 'abc=def') - def test_compat_shlex_split(self): - self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two']) - self.assertEqual(compat_shlex_split('-option "one\ntwo" \n -flag'), ['-option', 'one\ntwo', '-flag']) - self.assertEqual(compat_shlex_split('-val 中文'), ['-val', '中文']) - - def test_compat_etree_Element(self): - try: - compat_etree_Element.items - except AttributeError: - self.fail('compat_etree_Element is not a type') - def test_compat_etree_fromstring(self): xml = ''' <root foo="bar" spam="中文"> @@ -128,12 +81,12 @@ class TestCompat(unittest.TestCase): <foo><bar>spam</bar></foo> </root> ''' - doc = compat_etree_fromstring(xml.encode('utf-8')) - self.assertTrue(isinstance(doc.attrib['foo'], compat_str)) - self.assertTrue(isinstance(doc.attrib['spam'], compat_str)) - self.assertTrue(isinstance(doc.find('normal').text, compat_str)) - self.assertTrue(isinstance(doc.find('chinese').text, compat_str)) - self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str)) + doc = compat_etree_fromstring(xml.encode()) + self.assertTrue(isinstance(doc.attrib['foo'], str)) + self.assertTrue(isinstance(doc.attrib['spam'], str)) + self.assertTrue(isinstance(doc.find('normal').text, str)) + self.assertTrue(isinstance(doc.find('chinese').text, str)) + self.assertTrue(isinstance(doc.find('foo/bar').text, str)) def test_compat_etree_fromstring_doctype(self): xml = '''<?xml version="1.0"?> @@ -142,7 +95,7 @@ class TestCompat(unittest.TestCase): compat_etree_fromstring(xml) def test_struct_unpack(self): - self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,)) + self.assertEqual(struct.unpack('!B', b'\x00'), (0,)) if __name__ == '__main__': diff --git a/test/test_cookies.py b/test/test_cookies.py index 053e45b..ab5dd02 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -3,27 +3,28 @@ from datetime import datetime, timezone from hypervideo_dl import cookies from hypervideo_dl.cookies import ( + LenientSimpleCookie, LinuxChromeCookieDecryptor, MacChromeCookieDecryptor, WindowsChromeCookieDecryptor, - parse_safari_cookies, - pbkdf2_sha1, _get_linux_desktop_environment, _LinuxDesktopEnvironment, + parse_safari_cookies, + pbkdf2_sha1, ) class Logger: - def debug(self, message): + def debug(self, message, *args, **kwargs): print(f'[verbose] {message}') - def info(self, message): + def info(self, message, *args, **kwargs): print(message) - def warning(self, message, only_once=False): + def warning(self, message, *args, **kwargs): self.error(message) - def error(self, message): + def error(self, message, *args, **kwargs): raise Exception(message) @@ -137,3 +138,163 @@ class TestCookies(unittest.TestCase): def test_pbkdf2_sha1(self): key = pbkdf2_sha1(b'peanuts', b' ' * 16, 1, 16) self.assertEqual(key, b'g\xe1\x8e\x0fQ\x1c\x9b\xf3\xc9`!\xaa\x90\xd9\xd34') + + +class TestLenientSimpleCookie(unittest.TestCase): + def _run_tests(self, *cases): + for message, raw_cookie, expected in cases: + cookie = LenientSimpleCookie(raw_cookie) + + with self.subTest(message, expected=expected): + self.assertEqual(cookie.keys(), expected.keys(), message) + + for key, expected_value in expected.items(): + morsel = cookie[key] + if isinstance(expected_value, tuple): + expected_value, expected_attributes = expected_value + else: + expected_attributes = {} + + attributes = { + key: value + for key, value in dict(morsel).items() + if value != "" + } + self.assertEqual(attributes, expected_attributes, message) + + self.assertEqual(morsel.value, expected_value, message) + + def test_parsing(self): + self._run_tests( + # Copied from https://github.com/python/cpython/blob/v3.10.7/Lib/test/test_http_cookies.py + ( + "Test basic cookie", + "chips=ahoy; vienna=finger", + {"chips": "ahoy", "vienna": "finger"}, + ), + ( + "Test quoted cookie", + 'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"', + {"keebler": 'E=mc2; L="Loves"; fudge=\012;'}, + ), + ( + "Allow '=' in an unquoted value", + "keebler=E=mc2", + {"keebler": "E=mc2"}, + ), + ( + "Allow cookies with ':' in their name", + "key:term=value:term", + {"key:term": "value:term"}, + ), + ( + "Allow '[' and ']' in cookie values", + "a=b; c=[; d=r; f=h", + {"a": "b", "c": "[", "d": "r", "f": "h"}, + ), + ( + "Test basic cookie attributes", + 'Customer="WILE_E_COYOTE"; Version=1; Path=/acme', + {"Customer": ("WILE_E_COYOTE", {"version": "1", "path": "/acme"})}, + ), + ( + "Test flag only cookie attributes", + 'Customer="WILE_E_COYOTE"; HttpOnly; Secure', + {"Customer": ("WILE_E_COYOTE", {"httponly": True, "secure": True})}, + ), + ( + "Test flag only attribute with values", + "eggs=scrambled; httponly=foo; secure=bar; Path=/bacon", + {"eggs": ("scrambled", {"httponly": "foo", "secure": "bar", "path": "/bacon"})}, + ), + ( + "Test special case for 'expires' attribute, 4 digit year", + 'Customer="W"; expires=Wed, 01 Jan 2010 00:00:00 GMT', + {"Customer": ("W", {"expires": "Wed, 01 Jan 2010 00:00:00 GMT"})}, + ), + ( + "Test special case for 'expires' attribute, 2 digit year", + 'Customer="W"; expires=Wed, 01 Jan 98 00:00:00 GMT', + {"Customer": ("W", {"expires": "Wed, 01 Jan 98 00:00:00 GMT"})}, + ), + ( + "Test extra spaces in keys and values", + "eggs = scrambled ; secure ; path = bar ; foo=foo ", + {"eggs": ("scrambled", {"secure": True, "path": "bar"}), "foo": "foo"}, + ), + ( + "Test quoted attributes", + 'Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"', + {"Customer": ("WILE_E_COYOTE", {"version": "1", "path": "/acme"})} + ), + # Our own tests that CPython passes + ( + "Allow ';' in quoted value", + 'chips="a;hoy"; vienna=finger', + {"chips": "a;hoy", "vienna": "finger"}, + ), + ( + "Keep only the last set value", + "a=c; a=b", + {"a": "b"}, + ), + ) + + def test_lenient_parsing(self): + self._run_tests( + ( + "Ignore and try to skip invalid cookies", + 'chips={"ahoy;": 1}; vienna="finger;"', + {"vienna": "finger;"}, + ), + ( + "Ignore cookies without a name", + "a=b; unnamed; c=d", + {"a": "b", "c": "d"}, + ), + ( + "Ignore '\"' cookie without name", + 'a=b; "; c=d', + {"a": "b", "c": "d"}, + ), + ( + "Skip all space separated values", + "x a=b c=d x; e=f", + {"a": "b", "c": "d", "e": "f"}, + ), + ( + "Skip all space separated values", + 'x a=b; data={"complex": "json", "with": "key=value"}; x c=d x', + {"a": "b", "c": "d"}, + ), + ( + "Expect quote mending", + 'a=b; invalid="; c=d', + {"a": "b", "c": "d"}, + ), + ( + "Reset morsel after invalid to not capture attributes", + "a=b; invalid; Version=1; c=d", + {"a": "b", "c": "d"}, + ), + ( + "Reset morsel after invalid to not capture attributes", + "a=b; $invalid; $Version=1; c=d", + {"a": "b", "c": "d"}, + ), + ( + "Continue after non-flag attribute without value", + "a=b; path; Version=1; c=d", + {"a": "b", "c": "d"}, + ), + ( + "Allow cookie attributes with `$` prefix", + 'Customer="WILE_E_COYOTE"; $Version=1; $Secure; $Path=/acme', + {"Customer": ("WILE_E_COYOTE", {"version": "1", "secure": True, "path": "/acme"})}, + ), + ( + "Invalid Morsel keys should not result in an error", + "Key=Value; [Invalid]=Value; Another=Value", + {"Key": "Value", "Another": "Value"}, + ), + ) diff --git a/test/test_download.py b/test/test_download.py index 3cca13b..6f77343 100755 --- a/test/test_download.py +++ b/test/test_download.py @@ -1,43 +1,41 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import collections +import hashlib +import http.client +import json +import socket +import urllib.error + from test.helper import ( assertGreaterEqual, expect_info_dict, expect_warnings, get_params, gettestcases, + getwebpagetestcases, is_download_test, report_warning, try_rm, ) - -import hashlib -import io -import json -import socket - -import hypervideo_dl.YoutubeDL -from hypervideo_dl.compat import ( - compat_http_client, - compat_urllib_error, - compat_HTTPError, -) +import hypervideo_dl.YoutubeDL # isort: split +from hypervideo_dl.extractor import get_info_extractor from hypervideo_dl.utils import ( DownloadError, ExtractorError, - format_bytes, UnavailableVideoError, + format_bytes, + join_nonempty, ) -from hypervideo_dl.extractor import get_info_extractor RETRIES = 3 @@ -46,15 +44,15 @@ class YoutubeDL(hypervideo_dl.YoutubeDL): def __init__(self, *args, **kwargs): self.to_stderr = self.to_screen self.processed_info_dicts = [] - super(YoutubeDL, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) - def report_warning(self, message): + def report_warning(self, message, *args, **kwargs): # Don't accept warnings during tests raise ExtractorError(message) def process_info(self, info_dict): self.processed_info_dicts.append(info_dict.copy()) - return super(YoutubeDL, self).process_info(info_dict) + return super().process_info(info_dict) def _file_md5(fn): @@ -62,7 +60,9 @@ def _file_md5(fn): return hashlib.md5(f.read()).hexdigest() -defs = gettestcases() +normal_test_cases = gettestcases() +webpage_test_cases = getwebpagetestcases() +tests_counter = collections.defaultdict(collections.Counter) @is_download_test @@ -77,24 +77,13 @@ class TestDownload(unittest.TestCase): def __str__(self): """Identify each test with the `add_ie` attribute, if available.""" + cls, add_ie = type(self), getattr(self, self._testMethodName).add_ie + return f'{self._testMethodName} ({cls.__module__}.{cls.__name__}){f" [{add_ie}]" if add_ie else ""}:' - def strclass(cls): - """From 2.7's unittest; 2.6 had _strclass so we can't import it.""" - return '%s.%s' % (cls.__module__, cls.__name__) - - add_ie = getattr(self, self._testMethodName).add_ie - return '%s (%s)%s:' % (self._testMethodName, - strclass(self.__class__), - ' [%s]' % add_ie if add_ie else '') - - def setUp(self): - self.defs = defs # Dynamically generate tests - def generator(test_case, tname): - def test_template(self): if self.COMPLETED_TESTS.get(tname): return @@ -107,33 +96,34 @@ def generator(test_case, tname): def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) + self.skipTest(reason) + if not ie.working(): print_skipping('IE marked as not _WORKING') - return for tc in test_cases: info_dict = tc.get('info_dict', {}) params = tc.get('params', {}) if not info_dict.get('id'): - raise Exception('Test definition incorrect. \'id\' key is not present') - elif not info_dict.get('ext'): + raise Exception(f'Test {tname} definition incorrect - "id" key is not present') + elif not info_dict.get('ext') and info_dict.get('_type', 'video') == 'video': if params.get('skip_download') and params.get('ignore_no_formats_error'): continue - raise Exception('Test definition incorrect. The output file cannot be known. \'ext\' key is not present') + raise Exception(f'Test {tname} definition incorrect - "ext" key must be present to define the output file') if 'skip' in test_case: print_skipping(test_case['skip']) - return + for other_ie in other_ies: if not other_ie.working(): print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) - return params = get_params(test_case.get('params', {})) params['outtmpl'] = tname + '_' + params['outtmpl'] if is_playlist and 'playlist' not in test_case: params.setdefault('extract_flat', 'in_playlist') - params.setdefault('playlistend', test_case.get('playlist_mincount')) + params.setdefault('playlistend', test_case.get( + 'playlist_mincount', test_case.get('playlist_count', -2) + 1)) params.setdefault('skip_download', True) ydl = YoutubeDL(params, auto_init=False) @@ -172,14 +162,16 @@ def generator(test_case, tname): force_generic_extractor=params.get('force_generic_extractor', False)) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one - if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): + if (err.exc_info[0] not in (urllib.error.URLError, socket.timeout, UnavailableVideoError, http.client.BadStatusLine) + or (err.exc_info[0] == urllib.error.HTTPError and err.exc_info[1].code == 503)): + err.msg = f'{getattr(err, "msg", err)} ({tname})' raise if try_num == RETRIES: report_warning('%s failed due to network errors, skipping...' % tname) return - print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num)) + print(f'Retrying: {try_num} failed tries\n\n##########\n\n') try_num += 1 else: @@ -221,6 +213,8 @@ def generator(test_case, tname): tc_res_dict = res_dict['entries'][tc_num] # First, check test cases' data against extracted data alone expect_info_dict(self, tc_res_dict, tc.get('info_dict', {})) + if tc_res_dict.get('_type', 'video') != 'video': + continue # Now, check downloaded file consistency tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): @@ -245,7 +239,7 @@ def generator(test_case, tname): self.assertTrue( os.path.exists(info_json_fn), 'Missing info file %s' % info_json_fn) - with io.open(info_json_fn, encoding='utf-8') as infof: + with open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, info_dict, tc.get('info_dict', {})) finally: @@ -260,35 +254,43 @@ def generator(test_case, tname): # And add them to TestDownload -tests_counter = {} -for test_case in defs: - name = test_case['name'] - i = tests_counter.get(name, 0) - tests_counter[name] = i + 1 - tname = f'test_{name}_{i}' if i else f'test_{name}' - test_method = generator(test_case, tname) - test_method.__name__ = str(tname) - ie_list = test_case.get('add_ie') - test_method.add_ie = ie_list and ','.join(ie_list) - setattr(TestDownload, test_method.__name__, test_method) - del test_method +def inject_tests(test_cases, label=''): + for test_case in test_cases: + name = test_case['name'] + tname = join_nonempty('test', name, label, tests_counter[name][label], delim='_') + tests_counter[name][label] += 1 + + test_method = generator(test_case, tname) + test_method.__name__ = tname + test_method.add_ie = ','.join(test_case.get('add_ie', [])) + setattr(TestDownload, test_method.__name__, test_method) -def batch_generator(name, num_tests): +inject_tests(normal_test_cases) +# TODO: disable redirection to the IE to ensure we are actually testing the webpage extraction +inject_tests(webpage_test_cases, 'webpage') + + +def batch_generator(name): def test_template(self): - for i in range(num_tests): - getattr(self, f'test_{name}_{i}' if i else f'test_{name}')() + for label, num_tests in tests_counter[name].items(): + for i in range(num_tests): + test_name = join_nonempty('test', name, label, i, delim='_') + try: + getattr(self, test_name)() + except unittest.SkipTest: + print(f'Skipped {test_name}') return test_template -for name, num_tests in tests_counter.items(): - test_method = batch_generator(name, num_tests) +for name in tests_counter: + test_method = batch_generator(name) test_method.__name__ = f'test_{name}_all' test_method.add_ie = '' setattr(TestDownload, test_method.__name__, test_method) - del test_method +del test_method if __name__ == '__main__': diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py index 81b7dee..3b65859 100644 --- a/test/test_downloader_http.py +++ b/test/test_downloader_http.py @@ -1,20 +1,21 @@ #!/usr/bin/env python3 -# coding: utf-8 -from __future__ import unicode_literals # Allow direct execution import os -import re import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import http.server +import re +import threading + from test.helper import http_server_port, try_rm from hypervideo_dl import YoutubeDL -from hypervideo_dl.compat import compat_http_server from hypervideo_dl.downloader.http import HttpFD from hypervideo_dl.utils import encodeFilename -import threading TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -22,7 +23,7 @@ TEST_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_SIZE = 10 * 1024 -class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): +class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler): def log_message(self, format, *args): pass @@ -66,7 +67,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): assert False -class FakeLogger(object): +class FakeLogger: def debug(self, msg): pass @@ -79,7 +80,7 @@ class FakeLogger(object): class TestHttpFD(unittest.TestCase): def setUp(self): - self.httpd = compat_http_server.HTTPServer( + self.httpd = http.server.HTTPServer( ('127.0.0.1', 0), HTTPTestRequestHandler) self.port = http_server_port(self.httpd) self.server_thread = threading.Thread(target=self.httpd.serve_forever) @@ -94,8 +95,8 @@ class TestHttpFD(unittest.TestCase): try_rm(encodeFilename(filename)) self.assertTrue(downloader.real_download(filename, { 'url': 'http://127.0.0.1:%d/%s' % (self.port, ep), - })) - self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE) + }), ep) + self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep) try_rm(encodeFilename(filename)) def download_all(self, params): diff --git a/test/test_execution.py b/test/test_execution.py index d9aa965..56a4b2e 100644 --- a/test/test_execution.py +++ b/test/test_execution.py @@ -1,53 +1,56 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals +# Allow direct execution +import os +import sys import unittest -import sys -import os -import subprocess sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from hypervideo_dl.utils import encodeArgument -rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +import contextlib +import subprocess +from hypervideo_dl.utils import Popen -try: - _DEV_NULL = subprocess.DEVNULL -except AttributeError: - _DEV_NULL = open(os.devnull, 'wb') +rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +LAZY_EXTRACTORS = 'hypervideo_dl/extractor/lazy_extractors.py' class TestExecution(unittest.TestCase): + def run_hypervideo_dl(self, exe=(sys.executable, 'hypervideo_dl/__main__.py'), opts=('--version', )): + stdout, stderr, returncode = Popen.run( + [*exe, '--ignore-config', *opts], cwd=rootDir, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(stderr, file=sys.stderr) + self.assertEqual(returncode, 0) + return stdout.strip(), stderr.strip() + + def test_main_exec(self): + self.run_hypervideo_dl() + def test_import(self): - subprocess.check_call([sys.executable, '-c', 'import hypervideo_dl'], cwd=rootDir) + self.run_hypervideo_dl(exe=(sys.executable, '-c', 'import hypervideo_dl')) def test_module_exec(self): - if sys.version_info >= (2, 7): # Python 2.6 doesn't support package execution - subprocess.check_call([sys.executable, '-m', 'hypervideo_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL) - - def test_main_exec(self): - subprocess.check_call([sys.executable, 'hypervideo_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) + self.run_hypervideo_dl(exe=(sys.executable, '-m', 'hypervideo_dl')) def test_cmdline_umlauts(self): - p = subprocess.Popen( - [sys.executable, 'hypervideo_dl/__main__.py', encodeArgument('ä'), '--version'], - cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE) - _, stderr = p.communicate() + _, stderr = self.run_hypervideo_dl(opts=('ä', '--version')) self.assertFalse(stderr) def test_lazy_extractors(self): try: - subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'hypervideo_dl/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL) - subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL) + subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', LAZY_EXTRACTORS], + cwd=rootDir, stdout=subprocess.DEVNULL) + self.assertTrue(os.path.exists(LAZY_EXTRACTORS)) + + _, stderr = self.run_hypervideo_dl(opts=('-s', 'test:')) + self.assertFalse(stderr) + + subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=subprocess.DEVNULL) finally: - try: - os.remove('hypervideo_dl/extractor/lazy_extractors.py') - except (IOError, OSError): - pass + with contextlib.suppress(OSError): + os.remove(LAZY_EXTRACTORS) if __name__ == '__main__': diff --git a/test/test_http.py b/test/test_http.py index a7656b0..71d6f1b 100644 --- a/test/test_http.py +++ b/test/test_http.py @@ -1,23 +1,25 @@ #!/usr/bin/env python3 -# coding: utf-8 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import http_server_port -from hypervideo_dl import YoutubeDL -from hypervideo_dl.compat import compat_http_server, compat_urllib_request + +import http.server import ssl import threading +import urllib.request + +from test.helper import http_server_port +from hypervideo_dl import YoutubeDL TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): +class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler): def log_message(self, format, *args): pass @@ -32,17 +34,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): self.send_header('Content-Type', 'video/mp4') self.end_headers() self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]') - elif self.path == '/302': - if sys.version_info[0] == 3: - # XXX: Python 3 http server does not allow non-ASCII header values - self.send_response(404) - self.end_headers() - return - - new_url = 'http://127.0.0.1:%d/中文.html' % http_server_port(self.server) - self.send_response(302) - self.send_header(b'Location', new_url.encode('utf-8')) - self.end_headers() elif self.path == '/%E4%B8%AD%E6%96%87.html': self.send_response(200) self.send_header('Content-Type', 'text/html; charset=utf-8') @@ -52,7 +43,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): assert False -class FakeLogger(object): +class FakeLogger: def debug(self, msg): pass @@ -65,49 +56,84 @@ class FakeLogger(object): class TestHTTP(unittest.TestCase): def setUp(self): - self.httpd = compat_http_server.HTTPServer( + self.httpd = http.server.HTTPServer( ('127.0.0.1', 0), HTTPTestRequestHandler) self.port = http_server_port(self.httpd) self.server_thread = threading.Thread(target=self.httpd.serve_forever) self.server_thread.daemon = True self.server_thread.start() - def test_unicode_path_redirection(self): - # XXX: Python 3 http server does not allow non-ASCII header values - if sys.version_info[0] == 3: - return - - ydl = YoutubeDL({'logger': FakeLogger()}) - r = ydl.extract_info('http://127.0.0.1:%d/302' % self.port) - self.assertEqual(r['entries'][0]['url'], 'http://127.0.0.1:%d/vid.mp4' % self.port) - class TestHTTPS(unittest.TestCase): def setUp(self): certfn = os.path.join(TEST_DIR, 'testcert.pem') - self.httpd = compat_http_server.HTTPServer( + self.httpd = http.server.HTTPServer( ('127.0.0.1', 0), HTTPTestRequestHandler) - self.httpd.socket = ssl.wrap_socket( - self.httpd.socket, certfile=certfn, server_side=True) + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.load_cert_chain(certfn, None) + self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True) self.port = http_server_port(self.httpd) self.server_thread = threading.Thread(target=self.httpd.serve_forever) self.server_thread.daemon = True self.server_thread.start() def test_nocheckcertificate(self): - if sys.version_info >= (2, 7, 9): # No certificate checking anyways - ydl = YoutubeDL({'logger': FakeLogger()}) - self.assertRaises( - Exception, - ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port) + ydl = YoutubeDL({'logger': FakeLogger()}) + self.assertRaises( + Exception, + ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port) ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True}) r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port) - self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port) + self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port) + + +class TestClientCert(unittest.TestCase): + def setUp(self): + certfn = os.path.join(TEST_DIR, 'testcert.pem') + self.certdir = os.path.join(TEST_DIR, 'testdata', 'certificate') + cacertfn = os.path.join(self.certdir, 'ca.crt') + self.httpd = http.server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler) + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sslctx.verify_mode = ssl.CERT_REQUIRED + sslctx.load_verify_locations(cafile=cacertfn) + sslctx.load_cert_chain(certfn, None) + self.httpd.socket = sslctx.wrap_socket(self.httpd.socket, server_side=True) + self.port = http_server_port(self.httpd) + self.server_thread = threading.Thread(target=self.httpd.serve_forever) + self.server_thread.daemon = True + self.server_thread.start() + + def _run_test(self, **params): + ydl = YoutubeDL({ + 'logger': FakeLogger(), + # Disable client-side validation of unacceptable self-signed testcert.pem + # The test is of a check on the server side, so unaffected + 'nocheckcertificate': True, + **params, + }) + r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port) + self.assertEqual(r['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port) + + def test_certificate_combined_nopass(self): + self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithkey.crt')) + + def test_certificate_nocombined_nopass(self): + self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'), + client_certificate_key=os.path.join(self.certdir, 'client.key')) + + def test_certificate_combined_pass(self): + self._run_test(client_certificate=os.path.join(self.certdir, 'clientwithencryptedkey.crt'), + client_certificate_password='foobar') + + def test_certificate_nocombined_pass(self): + self._run_test(client_certificate=os.path.join(self.certdir, 'client.crt'), + client_certificate_key=os.path.join(self.certdir, 'clientencrypted.key'), + client_certificate_password='foobar') def _build_proxy_handler(name): - class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): + class HTTPTestRequestHandler(http.server.BaseHTTPRequestHandler): proxy_name = name def log_message(self, format, *args): @@ -117,20 +143,20 @@ def _build_proxy_handler(name): self.send_response(200) self.send_header('Content-Type', 'text/plain; charset=utf-8') self.end_headers() - self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8')) + self.wfile.write(f'{self.proxy_name}: {self.path}'.encode()) return HTTPTestRequestHandler class TestProxy(unittest.TestCase): def setUp(self): - self.proxy = compat_http_server.HTTPServer( + self.proxy = http.server.HTTPServer( ('127.0.0.1', 0), _build_proxy_handler('normal')) self.port = http_server_port(self.proxy) self.proxy_thread = threading.Thread(target=self.proxy.serve_forever) self.proxy_thread.daemon = True self.proxy_thread.start() - self.geo_proxy = compat_http_server.HTTPServer( + self.geo_proxy = http.server.HTTPServer( ('127.0.0.1', 0), _build_proxy_handler('geo')) self.geo_port = http_server_port(self.geo_proxy) self.geo_proxy_thread = threading.Thread(target=self.geo_proxy.serve_forever) @@ -138,26 +164,26 @@ class TestProxy(unittest.TestCase): self.geo_proxy_thread.start() def test_proxy(self): - geo_proxy = '127.0.0.1:{0}'.format(self.geo_port) + geo_proxy = f'127.0.0.1:{self.geo_port}' ydl = YoutubeDL({ - 'proxy': '127.0.0.1:{0}'.format(self.port), + 'proxy': f'127.0.0.1:{self.port}', 'geo_verification_proxy': geo_proxy, }) url = 'http://foo.com/bar' - response = ydl.urlopen(url).read().decode('utf-8') - self.assertEqual(response, 'normal: {0}'.format(url)) + response = ydl.urlopen(url).read().decode() + self.assertEqual(response, f'normal: {url}') - req = compat_urllib_request.Request(url) + req = urllib.request.Request(url) req.add_header('Ytdl-request-proxy', geo_proxy) - response = ydl.urlopen(req).read().decode('utf-8') - self.assertEqual(response, 'geo: {0}'.format(url)) + response = ydl.urlopen(req).read().decode() + self.assertEqual(response, f'geo: {url}') def test_proxy_with_idn(self): ydl = YoutubeDL({ - 'proxy': '127.0.0.1:{0}'.format(self.port), + 'proxy': f'127.0.0.1:{self.port}', }) url = 'http://中文.tw/' - response = ydl.urlopen(url).read().decode('utf-8') + response = ydl.urlopen(url).read().decode() # b'xn--fiq228c' is '中文'.encode('idna') self.assertEqual(response, 'normal: http://xn--fiq228c.tw/') diff --git a/test/test_netrc.py b/test/test_netrc.py index c7f5272..1d722ab 100644 --- a/test/test_netrc.py +++ b/test/test_netrc.py @@ -1,9 +1,10 @@ -# coding: utf-8 -from __future__ import unicode_literals +#!/usr/bin/env python3 +# Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/test/test_options.py b/test/test_options.py deleted file mode 100644 index 0b2458e..0000000 --- a/test/test_options.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding: utf-8 - -from __future__ import unicode_literals - -# Allow direct execution -import os -import sys -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from hypervideo_dl.options import _hide_login_info - - -class TestOptions(unittest.TestCase): - def test_hide_login_info(self): - self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']), - ['-u', 'PRIVATE', '-p', 'PRIVATE']) - self.assertEqual(_hide_login_info(['-u']), ['-u']) - self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']), - ['-u', 'PRIVATE', '-u', 'PRIVATE']) - self.assertEqual(_hide_login_info(['--username=foo']), - ['--username=PRIVATE']) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_overwrites.py b/test/test_overwrites.py index 9ad9bba..77fd100 100644 --- a/test/test_overwrites.py +++ b/test/test_overwrites.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals +# Allow direct execution import os -from os.path import join -import subprocess import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import is_download_test, try_rm +import subprocess + +from test.helper import is_download_test, try_rm root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -download_file = join(root_dir, 'test.webm') +download_file = os.path.join(root_dir, 'test.webm') @is_download_test @@ -46,7 +47,7 @@ class TestOverwrites(unittest.TestCase): self.assertTrue(os.path.getsize(download_file) > 1) def tearDown(self): - try_rm(join(root_dir, 'test.webm')) + try_rm(os.path.join(root_dir, 'test.webm')) if __name__ == '__main__': diff --git a/test/test_post_hooks.py b/test/test_post_hooks.py index 8f3b03a..61c35d7 100644 --- a/test/test_post_hooks.py +++ b/test/test_post_hooks.py @@ -1,20 +1,21 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - +# Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_params, try_rm, is_download_test -import hypervideo_dl.YoutubeDL + +from test.helper import get_params, is_download_test, try_rm +import hypervideo_dl.YoutubeDL # isort: split from hypervideo_dl.utils import DownloadError class YoutubeDL(hypervideo_dl.YoutubeDL): def __init__(self, *args, **kwargs): - super(YoutubeDL, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.to_stderr = self.to_screen diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py index e0b8347..982c321 100644 --- a/test/test_postprocessors.py +++ b/test/test_postprocessors.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals - # Allow direct execution import os import sys @@ -9,6 +7,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from hypervideo_dl import YoutubeDL from hypervideo_dl.compat import compat_shlex_quote from hypervideo_dl.postprocessor import ( @@ -16,7 +15,8 @@ from hypervideo_dl.postprocessor import ( FFmpegThumbnailsConvertorPP, MetadataFromFieldPP, MetadataParserPP, - ModifyChaptersPP + ModifyChaptersPP, + SponsorBlockPP, ) @@ -77,11 +77,15 @@ class TestModifyChaptersPP(unittest.TestCase): self._pp = ModifyChaptersPP(YoutubeDL()) @staticmethod - def _sponsor_chapter(start, end, cat, remove=False): - c = {'start_time': start, 'end_time': end, '_categories': [(cat, start, end)]} - if remove: - c['remove'] = True - return c + def _sponsor_chapter(start, end, cat, remove=False, title=None): + if title is None: + title = SponsorBlockPP.CATEGORIES[cat] + return { + 'start_time': start, + 'end_time': end, + '_categories': [(cat, start, end, title)], + **({'remove': True} if remove else {}), + } @staticmethod def _chapter(start, end, title=None, remove=False): @@ -131,6 +135,19 @@ class TestModifyChaptersPP(unittest.TestCase): 'c', '[SponsorBlock]: Filler Tangent', 'c']) self._remove_marked_arrange_sponsors_test_impl(chapters, expected, []) + def test_remove_marked_arrange_sponsors_SponsorBlockChapters(self): + chapters = self._chapters([70], ['c']) + [ + self._sponsor_chapter(10, 20, 'chapter', title='sb c1'), + self._sponsor_chapter(15, 16, 'chapter', title='sb c2'), + self._sponsor_chapter(30, 40, 'preview'), + self._sponsor_chapter(50, 60, 'filler')] + expected = self._chapters( + [10, 15, 16, 20, 30, 40, 50, 60, 70], + ['c', '[SponsorBlock]: sb c1', '[SponsorBlock]: sb c1, sb c2', '[SponsorBlock]: sb c1', + 'c', '[SponsorBlock]: Preview/Recap', + 'c', '[SponsorBlock]: Filler Tangent', 'c']) + self._remove_marked_arrange_sponsors_test_impl(chapters, expected, []) + def test_remove_marked_arrange_sponsors_UniqueNamesForOverlappingSponsors(self): chapters = self._chapters([120], ['c']) + [ self._sponsor_chapter(10, 45, 'sponsor'), self._sponsor_chapter(20, 40, 'selfpromo'), @@ -174,7 +191,7 @@ class TestModifyChaptersPP(unittest.TestCase): self._remove_marked_arrange_sponsors_test_impl(chapters, expected, cuts) def test_remove_marked_arrange_sponsors_ChapterWithCutHidingSponsor(self): - cuts = [self._sponsor_chapter(20, 50, 'selpromo', remove=True)] + cuts = [self._sponsor_chapter(20, 50, 'selfpromo', remove=True)] chapters = self._chapters([60], ['c']) + [ self._sponsor_chapter(10, 20, 'intro'), self._sponsor_chapter(30, 40, 'sponsor'), @@ -200,7 +217,7 @@ class TestModifyChaptersPP(unittest.TestCase): self._sponsor_chapter(10, 20, 'sponsor'), self._sponsor_chapter(20, 30, 'interaction', remove=True), self._chapter(30, 40, remove=True), - self._sponsor_chapter(40, 50, 'selpromo', remove=True), + self._sponsor_chapter(40, 50, 'selfpromo', remove=True), self._sponsor_chapter(50, 60, 'interaction')] expected = self._chapters([10, 20, 30, 40], ['c', '[SponsorBlock]: Sponsor', @@ -283,7 +300,7 @@ class TestModifyChaptersPP(unittest.TestCase): chapters = self._chapters([70], ['c']) + [ self._sponsor_chapter(10, 30, 'sponsor'), self._sponsor_chapter(20, 50, 'interaction'), - self._sponsor_chapter(30, 50, 'selpromo', remove=True), + self._sponsor_chapter(30, 50, 'selfpromo', remove=True), self._sponsor_chapter(40, 60, 'sponsor'), self._sponsor_chapter(50, 60, 'interaction')] expected = self._chapters( diff --git a/test/test_socks.py b/test/test_socks.py index 2574e73..6651290 100644 --- a/test/test_socks.py +++ b/test/test_socks.py @@ -1,25 +1,18 @@ #!/usr/bin/env python3 -# coding: utf-8 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import random import subprocess +import urllib.request -from test.helper import ( - FakeYDL, - get_params, - is_download_test, -) -from hypervideo_dl.compat import ( - compat_str, - compat_urllib_request, -) +from test.helper import FakeYDL, get_params, is_download_test @is_download_test @@ -41,7 +34,7 @@ class TestMultipleSocks(unittest.TestCase): 'proxy': params['primary_proxy'] }) self.assertEqual( - ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8'), + ydl.urlopen('http://yt-dl.org/ip').read().decode(), params['primary_server_ip']) def test_proxy_https(self): @@ -52,7 +45,7 @@ class TestMultipleSocks(unittest.TestCase): 'proxy': params['primary_proxy'] }) self.assertEqual( - ydl.urlopen('https://yt-dl.org/ip').read().decode('utf-8'), + ydl.urlopen('https://yt-dl.org/ip').read().decode(), params['primary_server_ip']) def test_secondary_proxy_http(self): @@ -60,10 +53,10 @@ class TestMultipleSocks(unittest.TestCase): if params is None: return ydl = FakeYDL() - req = compat_urllib_request.Request('http://yt-dl.org/ip') + req = urllib.request.Request('http://yt-dl.org/ip') req.add_header('Ytdl-request-proxy', params['secondary_proxy']) self.assertEqual( - ydl.urlopen(req).read().decode('utf-8'), + ydl.urlopen(req).read().decode(), params['secondary_server_ip']) def test_secondary_proxy_https(self): @@ -71,10 +64,10 @@ class TestMultipleSocks(unittest.TestCase): if params is None: return ydl = FakeYDL() - req = compat_urllib_request.Request('https://yt-dl.org/ip') + req = urllib.request.Request('https://yt-dl.org/ip') req.add_header('Ytdl-request-proxy', params['secondary_proxy']) self.assertEqual( - ydl.urlopen(req).read().decode('utf-8'), + ydl.urlopen(req).read().decode(), params['secondary_server_ip']) @@ -105,16 +98,16 @@ class TestSocks(unittest.TestCase): ydl = FakeYDL({ 'proxy': '%s://127.0.0.1:%d' % (protocol, self.port), }) - return ydl.urlopen('http://yt-dl.org/ip').read().decode('utf-8') + return ydl.urlopen('http://yt-dl.org/ip').read().decode() def test_socks4(self): - self.assertTrue(isinstance(self._get_ip('socks4'), compat_str)) + self.assertTrue(isinstance(self._get_ip('socks4'), str)) def test_socks4a(self): - self.assertTrue(isinstance(self._get_ip('socks4a'), compat_str)) + self.assertTrue(isinstance(self._get_ip('socks4a'), str)) def test_socks5(self): - self.assertTrue(isinstance(self._get_ip('socks5'), compat_str)) + self.assertTrue(isinstance(self._get_ip('socks5'), str)) if __name__ == '__main__': diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 10fa0ca..7657d43 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -1,33 +1,32 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, md5, is_download_test +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from test.helper import FakeYDL, is_download_test, md5 from hypervideo_dl.extractor import ( - YoutubeIE, - DailymotionIE, - TedTalkIE, - VimeoIE, - WallaIE, - CeskaTelevizeIE, - LyndaIE, NPOIE, + NRKTVIE, PBSIE, + CeskaTelevizeIE, ComedyCentralIE, - NRKTVIE, + DailymotionIE, + DemocracynowIE, + LyndaIE, RaiPlayIE, - VikiIE, - ThePlatformIE, - ThePlatformFeedIE, RTVEALaCartaIE, - DemocracynowIE, + TedTalkIE, + ThePlatformFeedIE, + ThePlatformIE, + VikiIE, + VimeoIE, + WallaIE, + YoutubeIE, ) @@ -40,6 +39,9 @@ class BaseTestSubtitles(unittest.TestCase): self.DL = FakeYDL() self.ie = self.IE() self.DL.add_info_extractor(self.ie) + if not self.IE.working(): + print('Skipping: %s marked as not _WORKING' % self.IE.ie_key()) + self.skipTest('IE marked as not _WORKING') def getInfoDict(self): info_dict = self.DL.extract_info(self.url, download=False) @@ -53,12 +55,27 @@ class BaseTestSubtitles(unittest.TestCase): for sub_info in subtitles.values(): if sub_info.get('data') is None: uf = self.DL.urlopen(sub_info['url']) - sub_info['data'] = uf.read().decode('utf-8') - return dict((l, sub_info['data']) for l, sub_info in subtitles.items()) + sub_info['data'] = uf.read().decode() + return {l: sub_info['data'] for l, sub_info in subtitles.items()} @is_download_test class TestYoutubeSubtitles(BaseTestSubtitles): + # Available subtitles for QRS8MkLhQmM: + # Language formats + # ru vtt, ttml, srv3, srv2, srv1, json3 + # fr vtt, ttml, srv3, srv2, srv1, json3 + # en vtt, ttml, srv3, srv2, srv1, json3 + # nl vtt, ttml, srv3, srv2, srv1, json3 + # de vtt, ttml, srv3, srv2, srv1, json3 + # ko vtt, ttml, srv3, srv2, srv1, json3 + # it vtt, ttml, srv3, srv2, srv1, json3 + # zh-Hant vtt, ttml, srv3, srv2, srv1, json3 + # hi vtt, ttml, srv3, srv2, srv1, json3 + # pt-BR vtt, ttml, srv3, srv2, srv1, json3 + # es-MX vtt, ttml, srv3, srv2, srv1, json3 + # ja vtt, ttml, srv3, srv2, srv1, json3 + # pl vtt, ttml, srv3, srv2, srv1, json3 url = 'QRS8MkLhQmM' IE = YoutubeIE @@ -67,47 +84,60 @@ class TestYoutubeSubtitles(BaseTestSubtitles): self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() self.assertEqual(len(subtitles.keys()), 13) - self.assertEqual(md5(subtitles['en']), '688dd1ce0981683867e7fe6fde2a224b') - self.assertEqual(md5(subtitles['it']), '31324d30b8430b309f7f5979a504a769') + self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d') + self.assertEqual(md5(subtitles['it']), '0e0b667ba68411d88fd1c5f4f4eab2f9') for lang in ['fr', 'de']: self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang) - def test_youtube_subtitles_ttml_format(self): + def _test_subtitles_format(self, fmt, md5_hash, lang='en'): self.DL.params['writesubtitles'] = True - self.DL.params['subtitlesformat'] = 'ttml' + self.DL.params['subtitlesformat'] = fmt subtitles = self.getSubtitles() - self.assertEqual(md5(subtitles['en']), 'c97ddf1217390906fa9fbd34901f3da2') + self.assertEqual(md5(subtitles[lang]), md5_hash) + + def test_youtube_subtitles_ttml_format(self): + self._test_subtitles_format('ttml', 'c97ddf1217390906fa9fbd34901f3da2') def test_youtube_subtitles_vtt_format(self): - self.DL.params['writesubtitles'] = True - self.DL.params['subtitlesformat'] = 'vtt' - subtitles = self.getSubtitles() - self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d') + self._test_subtitles_format('vtt', 'ae1bd34126571a77aabd4d276b28044d') - def test_youtube_automatic_captions(self): - self.url = '8YoUxe5ncPo' - self.DL.params['writeautomaticsub'] = True - self.DL.params['subtitleslangs'] = ['it'] - subtitles = self.getSubtitles() - self.assertTrue(subtitles['it'] is not None) + def test_youtube_subtitles_json3_format(self): + self._test_subtitles_format('json3', '688dd1ce0981683867e7fe6fde2a224b') - def test_youtube_no_automatic_captions(self): - self.url = 'QRS8MkLhQmM' + def _test_automatic_captions(self, url, lang): + self.url = url self.DL.params['writeautomaticsub'] = True + self.DL.params['subtitleslangs'] = [lang] subtitles = self.getSubtitles() - self.assertTrue(not subtitles) + self.assertTrue(subtitles[lang] is not None) + def test_youtube_automatic_captions(self): + # Available automatic captions for 8YoUxe5ncPo: + # Language formats (all in vtt, ttml, srv3, srv2, srv1, json3) + # gu, zh-Hans, zh-Hant, gd, ga, gl, lb, la, lo, tt, tr, + # lv, lt, tk, th, tg, te, fil, haw, yi, ceb, yo, de, da, + # el, eo, en, eu, et, es, ru, rw, ro, bn, be, bg, uk, jv, + # bs, ja, or, xh, co, ca, cy, cs, ps, pt, pa, vi, pl, hy, + # hr, ht, hu, hmn, hi, ha, mg, uz, ml, mn, mi, mk, ur, + # mt, ms, mr, ug, ta, my, af, sw, is, am, + # *it*, iw, sv, ar, + # su, zu, az, id, ig, nl, no, ne, ny, fr, ku, fy, fa, fi, + # ka, kk, sr, sq, ko, kn, km, st, sk, si, so, sn, sm, sl, + # ky, sd + # ... + self._test_automatic_captions('8YoUxe5ncPo', 'it') + + @unittest.skip('Video unavailable') def test_youtube_translated_subtitles(self): - # This video has a subtitles track, which can be translated - self.url = 'i0ZabxXmH4Y' - self.DL.params['writeautomaticsub'] = True - self.DL.params['subtitleslangs'] = ['it'] - subtitles = self.getSubtitles() - self.assertTrue(subtitles['it'] is not None) + # This video has a subtitles track, which can be translated (#4555) + self._test_automatic_captions('Ky9eprVWzlI', 'it') def test_youtube_nosubtitles(self): self.DL.expect_warning('video doesn\'t have subtitles') - self.url = 'n5BB19UTcdA' + # Available automatic captions for 8YoUxe5ncPo: + # ... + # 8YoUxe5ncPo has no subtitles + self.url = '8YoUxe5ncPo' self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() @@ -139,6 +169,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles): @is_download_test +@unittest.skip('IE broken') class TestTedSubtitles(BaseTestSubtitles): url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html' IE = TedTalkIE @@ -163,13 +194,13 @@ class TestVimeoSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr'])) - self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888') - self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8') + self.assertEqual(set(subtitles.keys()), {'de', 'en', 'es', 'fr'}) + self.assertEqual(md5(subtitles['en']), '386cbc9320b94e25cb364b97935e5dd1') + self.assertEqual(md5(subtitles['fr']), 'c9b69eef35bc6641c0d4da8a04f9dfac') def test_nosubtitles(self): self.DL.expect_warning('video doesn\'t have subtitles') - self.url = 'http://vimeo.com/56015672' + self.url = 'http://vimeo.com/68093876' self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() @@ -177,6 +208,7 @@ class TestVimeoSubtitles(BaseTestSubtitles): @is_download_test +@unittest.skip('IE broken') class TestWallaSubtitles(BaseTestSubtitles): url = 'http://vod.walla.co.il/movie/2705958/the-yes-men' IE = WallaIE @@ -186,7 +218,7 @@ class TestWallaSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['heb'])) + self.assertEqual(set(subtitles.keys()), {'heb'}) self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920') def test_nosubtitles(self): @@ -199,6 +231,7 @@ class TestWallaSubtitles(BaseTestSubtitles): @is_download_test +@unittest.skip('IE broken') class TestCeskaTelevizeSubtitles(BaseTestSubtitles): url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky' IE = CeskaTelevizeIE @@ -208,7 +241,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['cs'])) + self.assertEqual(set(subtitles.keys()), {'cs'}) self.assertTrue(len(subtitles['cs']) > 20000) def test_nosubtitles(self): @@ -221,6 +254,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles): @is_download_test +@unittest.skip('IE broken') class TestLyndaSubtitles(BaseTestSubtitles): url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html' IE = LyndaIE @@ -229,11 +263,12 @@ class TestLyndaSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7') @is_download_test +@unittest.skip('IE broken') class TestNPOSubtitles(BaseTestSubtitles): url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860' IE = NPOIE @@ -242,23 +277,24 @@ class TestNPOSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['nl'])) + self.assertEqual(set(subtitles.keys()), {'nl'}) self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4') @is_download_test +@unittest.skip('IE broken') class TestMTVSubtitles(BaseTestSubtitles): url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans' IE = ComedyCentralIE def getInfoDict(self): - return super(TestMTVSubtitles, self).getInfoDict()['entries'][0] + return super().getInfoDict()['entries'][0] def test_allsubtitles(self): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) self.assertEqual(md5(subtitles['en']), '78206b8d8a0cfa9da64dc026eea48961') @@ -271,8 +307,8 @@ class TestNRKSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['no'])) - self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2') + self.assertEqual(set(subtitles.keys()), {'nb-ttv'}) + self.assertEqual(md5(subtitles['nb-ttv']), '67e06ff02d0deaf975e68f6cb8f6a149') @is_download_test @@ -284,7 +320,7 @@ class TestRaiPlaySubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['it'])) + self.assertEqual(set(subtitles.keys()), {'it'}) self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a') def test_subtitles_array_key(self): @@ -292,11 +328,12 @@ class TestRaiPlaySubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['it'])) + self.assertEqual(set(subtitles.keys()), {'it'}) self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd') @is_download_test +@unittest.skip('IE broken - DRM only') class TestVikiSubtitles(BaseTestSubtitles): url = 'http://www.viki.com/videos/1060846v-punch-episode-18' IE = VikiIE @@ -305,7 +342,7 @@ class TestVikiSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a') @@ -320,11 +357,12 @@ class TestThePlatformSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b') @is_download_test +@unittest.skip('IE broken') class TestThePlatformFeedSubtitles(BaseTestSubtitles): url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207' IE = ThePlatformFeedIE @@ -333,7 +371,7 @@ class TestThePlatformFeedSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) self.assertEqual(md5(subtitles['en']), '48649a22e82b2da21c9a67a395eedade') @@ -348,7 +386,7 @@ class TestRtveSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['es'])) + self.assertEqual(set(subtitles.keys()), {'es'}) self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca') @@ -361,16 +399,16 @@ class TestDemocracynowSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) - self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c') + self.assertEqual(set(subtitles.keys()), {'en'}) + self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045') def test_subtitles_in_page(self): self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree' self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) - self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c') + self.assertEqual(set(subtitles.keys()), {'en'}) + self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045') @is_download_test @@ -382,7 +420,7 @@ class TestPBSSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['allsubtitles'] = True subtitles = self.getSubtitles() - self.assertEqual(set(subtitles.keys()), set(['en'])) + self.assertEqual(set(subtitles.keys()), {'en'}) def test_subtitles_dfxp_format(self): self.DL.params['writesubtitles'] = True diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py deleted file mode 100644 index 6c1b7ec..0000000 --- a/test/test_unicode_literals.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import unicode_literals - -# Allow direct execution -import os -import sys -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import io -import re - -rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -IGNORED_FILES = [ - 'setup.py', # http://bugs.python.org/issue13943 - 'conf.py', - 'buildserver.py', -] - -IGNORED_DIRS = [ - '.git', - '.tox', -] - -from test.helper import assertRegexpMatches - - -class TestUnicodeLiterals(unittest.TestCase): - def test_all_files(self): - for dirpath, dirnames, filenames in os.walk(rootDir): - for ignore_dir in IGNORED_DIRS: - if ignore_dir in dirnames: - # If we remove the directory from dirnames os.walk won't - # recurse into it - dirnames.remove(ignore_dir) - for basename in filenames: - if not basename.endswith('.py'): - continue - if basename in IGNORED_FILES: - continue - - fn = os.path.join(dirpath, basename) - with io.open(fn, encoding='utf-8') as inf: - code = inf.read() - - if "'" not in code and '"' not in code: - continue - assertRegexpMatches( - self, - code, - r'(?:(?:#.*?|\s*)\n)*from __future__ import (?:[a-z_]+,\s*)*unicode_literals', - 'unicode_literals import missing in %s' % fn) - - m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code) - if m is not None: - self.assertTrue( - m is None, - 'u present in %s, around %s' % ( - fn, code[m.start() - 10:m.end() + 10])) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index 039900c..acb913a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,89 +1,108 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals # Allow direct execution import os +import re import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# Various small unit tests +import contextlib import io import itertools import json import xml.etree.ElementTree +from hypervideo_dl.compat import ( + compat_etree_fromstring, + compat_HTMLParseError, + compat_os_name, +) from hypervideo_dl.utils import ( + Config, + DateRange, + ExtractorError, + InAdvancePagedList, + LazyList, + OnDemandPagedList, age_restricted, args_to_str, - encode_base_n, + base_url, caesar, clean_html, clean_podcast_url, - Config, + cli_bool_option, + cli_option, + cli_valueless_option, date_from_str, datetime_from_str, - DateRange, detect_exe_version, determine_ext, + determine_file_encoding, + dfxp2srt, dict_get, + encode_base_n, encode_compat_str, encodeFilename, escape_rfc3986, escape_url, + expand_path, extract_attributes, - ExtractorError, find_xpath_attr, fix_xml_ampersands, - format_bytes, float_or_none, - get_element_by_class, + format_bytes, + get_compatible_ext, get_element_by_attribute, - get_elements_by_class, - get_elements_by_attribute, - get_element_html_by_class, + get_element_by_class, get_element_html_by_attribute, - get_elements_html_by_class, + get_element_html_by_class, + get_element_text_and_html_by_tag, + get_elements_by_attribute, + get_elements_by_class, get_elements_html_by_attribute, + get_elements_html_by_class, get_elements_text_and_html_by_attribute, - get_element_text_and_html_by_tag, - InAdvancePagedList, int_or_none, intlist_to_bytes, + iri_to_uri, is_html, js_to_json, limit_length, + locked_file, + lowercase_escape, + match_str, merge_dicts, mimetype2ext, month_by_name, multipart_encode, ohdave_rsa_encrypt, - OnDemandPagedList, orderedSet, parse_age_limit, + parse_bitrate, + parse_codecs, + parse_count, + parse_dfxp_time_expr, parse_duration, parse_filesize, - parse_count, parse_iso8601, - parse_resolution, - parse_bitrate, parse_qs, + parse_resolution, pkcs1pad, + prepend_extension, read_batch_urls, + remove_end, + remove_quotes, + remove_start, + render_table, + replace_extension, + rot47, sanitize_filename, sanitize_path, sanitize_url, sanitized_Request, - expand_path, - prepend_extension, - replace_extension, - remove_start, - remove_end, - remove_quotes, - rot47, shell_quote, smuggle_url, str_to_int, @@ -91,42 +110,23 @@ from hypervideo_dl.utils import ( strip_or_none, subtitles_filename, timeconvert, + traverse_obj, unescapeHTML, unified_strdate, unified_timestamp, unsmuggle_url, + update_url_query, uppercase_escape, - lowercase_escape, url_basename, url_or_none, - base_url, - urljoin, urlencode_postdata, + urljoin, urshift, - update_url_query, version_tuple, - xpath_with_ns, + xpath_attr, xpath_element, xpath_text, - xpath_attr, - render_table, - match_str, - parse_dfxp_time_expr, - dfxp2srt, - cli_option, - cli_valueless_option, - cli_bool_option, - parse_codecs, - iri_to_uri, - LazyList, -) -from hypervideo_dl.compat import ( - compat_chr, - compat_etree_fromstring, - compat_getenv, - compat_HTMLParseError, - compat_os_name, - compat_setenv, + xpath_with_ns, ) @@ -142,13 +142,13 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_filename('123'), '123') - self.assertEqual('abc_de', sanitize_filename('abc/de')) + self.assertEqual('abc⧸de', sanitize_filename('abc/de')) self.assertFalse('/' in sanitize_filename('abc/de///')) - self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de')) - self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|')) - self.assertEqual('yes no', sanitize_filename('yes? no')) - self.assertEqual('this - that', sanitize_filename('this: that')) + self.assertEqual('abc_de', sanitize_filename('abc/<>\\*|de', is_id=False)) + self.assertEqual('xxx', sanitize_filename('xxx/<>\\*|', is_id=False)) + self.assertEqual('yes no', sanitize_filename('yes? no', is_id=False)) + self.assertEqual('this - that', sanitize_filename('this: that', is_id=False)) self.assertEqual(sanitize_filename('AT&T'), 'AT&T') aumlaut = 'ä' @@ -265,15 +265,22 @@ class TestUtil(unittest.TestCase): def test_expand_path(self): def env(var): - return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var) + return f'%{var}%' if sys.platform == 'win32' else f'${var}' - compat_setenv('hypervideo_dl_EXPATH_PATH', 'expanded') + os.environ['hypervideo_dl_EXPATH_PATH'] = 'expanded' self.assertEqual(expand_path(env('hypervideo_dl_EXPATH_PATH')), 'expanded') - self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME')) - self.assertEqual(expand_path('~'), compat_getenv('HOME')) - self.assertEqual( - expand_path('~/%s' % env('hypervideo_dl_EXPATH_PATH')), - '%s/expanded' % compat_getenv('HOME')) + + old_home = os.environ.get('HOME') + test_str = R'C:\Documents and Settings\тест\Application Data' + try: + os.environ['HOME'] = test_str + self.assertEqual(expand_path(env('HOME')), os.getenv('HOME')) + self.assertEqual(expand_path('~'), os.getenv('HOME')) + self.assertEqual( + expand_path('~/%s' % env('hypervideo_dl_EXPATH_PATH')), + '%s/expanded' % os.getenv('HOME')) + finally: + os.environ['HOME'] = old_home or '' def test_prepend_extension(self): self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext') @@ -364,6 +371,7 @@ class TestUtil(unittest.TestCase): self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011') self.assertEqual(unified_strdate('1968 12 10'), '19681210') self.assertEqual(unified_strdate('1968-12-10'), '19681210') + self.assertEqual(unified_strdate('31-07-2022 20:00'), '20220731') self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128') self.assertEqual( unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False), @@ -407,6 +415,10 @@ class TestUtil(unittest.TestCase): self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140) self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363) + self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1) + self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86) + self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78) + def test_determine_ext(self): self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4') self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None) @@ -537,9 +549,6 @@ class TestUtil(unittest.TestCase): self.assertEqual(str_to_int('123,456'), 123456) self.assertEqual(str_to_int('123.456'), 123456) self.assertEqual(str_to_int(523), 523) - # Python 3 has no long - if sys.version_info < (3, 0): - eval('self.assertEqual(str_to_int(123456L), 123456)') self.assertEqual(str_to_int('noninteger'), None) self.assertEqual(str_to_int([]), None) @@ -559,6 +568,7 @@ class TestUtil(unittest.TestCase): self.assertEqual(base_url('http://foo.de/bar/'), 'http://foo.de/bar/') self.assertEqual(base_url('http://foo.de/bar/baz'), 'http://foo.de/bar/') self.assertEqual(base_url('http://foo.de/bar/baz?x=z/x/c'), 'http://foo.de/bar/') + self.assertEqual(base_url('http://foo.de/bar/baz&x=z&w=y/x/c'), 'http://foo.de/bar/baz&x=z&w=y/x/') def test_urljoin(self): self.assertEqual(urljoin('http://foo.de/', '/a/b/c.txt'), 'http://foo.de/a/b/c.txt') @@ -668,8 +678,7 @@ class TestUtil(unittest.TestCase): def get_page(pagenum): firstid = pagenum * pagesize upto = min(size, pagenum * pagesize + pagesize) - for i in range(firstid, upto): - yield i + yield from range(firstid, upto) pl = OnDemandPagedList(get_page, pagesize) got = pl.getslice(*sliceargs) @@ -738,7 +747,7 @@ class TestUtil(unittest.TestCase): multipart_encode({b'field': b'value'}, boundary='AAAAAA')[0], b'--AAAAAA\r\nContent-Disposition: form-data; name="field"\r\n\r\nvalue\r\n--AAAAAA--\r\n') self.assertEqual( - multipart_encode({'欄位'.encode('utf-8'): '值'.encode('utf-8')}, boundary='AAAAAA')[0], + multipart_encode({'欄位'.encode(): '值'.encode()}, boundary='AAAAAA')[0], b'--AAAAAA\r\nContent-Disposition: form-data; name="\xe6\xac\x84\xe4\xbd\x8d"\r\n\r\n\xe5\x80\xbc\r\n--AAAAAA--\r\n') self.assertRaises( ValueError, multipart_encode, {b'field': b'value'}, boundary='value') @@ -896,7 +905,7 @@ class TestUtil(unittest.TestCase): 'dynamic_range': 'HDR10', }) self.assertEqual(parse_codecs('av01.0.12M.10.0.110.09.16.09.0'), { - 'vcodec': 'av01.0.12M.10', + 'vcodec': 'av01.0.12M.10.0.110.09.16.09.0', 'acodec': 'none', 'dynamic_range': 'HDR10', }) @@ -1091,6 +1100,12 @@ class TestUtil(unittest.TestCase): on = js_to_json('[1,//{},\n2]') self.assertEqual(json.loads(on), [1, 2]) + on = js_to_json(R'"\^\$\#"') + self.assertEqual(json.loads(on), R'^$#', msg='Unnecessary escapes should be stripped') + + on = js_to_json('\'"\\""\'') + self.assertEqual(json.loads(on), '"""', msg='Unnecessary quote escape should be escaped') + def test_js_to_json_malformed(self): self.assertEqual(js_to_json('42a1'), '42"a1"') self.assertEqual(js_to_json('42a-1'), '42"a"-1') @@ -1126,7 +1141,7 @@ class TestUtil(unittest.TestCase): self.assertEqual(extract_attributes('<e x="décomposé">'), {'x': 'décompose\u0301'}) # "Narrow" Python builds don't support unicode code points outside BMP. try: - compat_chr(0x10000) + chr(0x10000) supports_outside_bmp = True except ValueError: supports_outside_bmp = False @@ -1399,7 +1414,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') <p begin="3" dur="-1">Ignored, three</p> </div> </body> - </tt>'''.encode('utf-8') + </tt>'''.encode() srt_data = '''1 00:00:00,000 --> 00:00:01,000 The following line contains Chinese characters and special symbols @@ -1417,14 +1432,14 @@ Line ''' self.assertEqual(dfxp2srt(dfxp_data), srt_data) - dfxp_data_no_default_namespace = '''<?xml version="1.0" encoding="UTF-8"?> + dfxp_data_no_default_namespace = b'''<?xml version="1.0" encoding="UTF-8"?> <tt xml:lang="en" xmlns:tts="http://www.w3.org/ns/ttml#parameter"> <body> <div xml:lang="en"> <p begin="0" end="1">The first line</p> </div> </body> - </tt>'''.encode('utf-8') + </tt>''' srt_data = '''1 00:00:00,000 --> 00:00:01,000 The first line @@ -1432,7 +1447,7 @@ The first line ''' self.assertEqual(dfxp2srt(dfxp_data_no_default_namespace), srt_data) - dfxp_data_with_style = '''<?xml version="1.0" encoding="utf-8"?> + dfxp_data_with_style = b'''<?xml version="1.0" encoding="utf-8"?> <tt xmlns="http://www.w3.org/2006/10/ttaf1" xmlns:ttp="http://www.w3.org/2006/10/ttaf1#parameter" ttp:timeBase="media" xmlns:tts="http://www.w3.org/2006/10/ttaf1#style" xml:lang="en" xmlns:ttm="http://www.w3.org/2006/10/ttaf1#metadata"> <head> <styling> @@ -1450,7 +1465,7 @@ The first line <p style="s1" tts:textDecoration="underline" begin="00:00:09.56" id="p2" end="00:00:12.36"><span style="s2" tts:color="lime">inner<br /> </span>style</p> </div> </body> -</tt>'''.encode('utf-8') +</tt>''' srt_data = '''1 00:00:02,080 --> 00:00:05,840 <font color="white" face="sansSerif" size="16">default style<font color="red">custom style</font></font> @@ -1670,6 +1685,9 @@ Line 1 self.assertEqual(list(get_elements_text_and_html_by_attribute('class', 'foo', html)), []) self.assertEqual(list(get_elements_text_and_html_by_attribute('class', 'no-such-foo', html)), []) + self.assertEqual(list(get_elements_text_and_html_by_attribute( + 'class', 'foo', '<a class="foo">nice</a><span class="foo">nice</span>', tag='a')), [('nice', '<a class="foo">nice</a>')]) + GET_ELEMENT_BY_TAG_TEST_STRING = ''' random text lorem ipsum</p> <div> @@ -1757,7 +1775,7 @@ Line 1 def test(ll, idx, val, cache): self.assertEqual(ll[idx], val) - self.assertEqual(getattr(ll, '_LazyList__cache'), list(cache)) + self.assertEqual(ll._cache, list(cache)) ll = LazyList(range(10)) test(ll, 0, 0, range(1)) @@ -1795,6 +1813,302 @@ Line 1 self.assertEqual(Config.hide_login_info(['--username=foo']), ['--username=PRIVATE']) + def test_locked_file(self): + TEXT = 'test_locked_file\n' + FILE = 'test_locked_file.ytdl' + MODES = 'war' # Order is important + + try: + for lock_mode in MODES: + with locked_file(FILE, lock_mode, False) as f: + if lock_mode == 'r': + self.assertEqual(f.read(), TEXT * 2, 'Wrong file content') + else: + f.write(TEXT) + for test_mode in MODES: + testing_write = test_mode != 'r' + try: + with locked_file(FILE, test_mode, False): + pass + except (BlockingIOError, PermissionError): + if not testing_write: # FIXME + print(f'Known issue: Exclusive lock ({lock_mode}) blocks read access ({test_mode})') + continue + self.assertTrue(testing_write, f'{test_mode} is blocked by {lock_mode}') + else: + self.assertFalse(testing_write, f'{test_mode} is not blocked by {lock_mode}') + finally: + with contextlib.suppress(OSError): + os.remove(FILE) + + def test_determine_file_encoding(self): + self.assertEqual(determine_file_encoding(b''), (None, 0)) + self.assertEqual(determine_file_encoding(b'--verbose -x --audio-format mkv\n'), (None, 0)) + + self.assertEqual(determine_file_encoding(b'\xef\xbb\xbf'), ('utf-8', 3)) + self.assertEqual(determine_file_encoding(b'\x00\x00\xfe\xff'), ('utf-32-be', 4)) + self.assertEqual(determine_file_encoding(b'\xff\xfe'), ('utf-16-le', 2)) + + self.assertEqual(determine_file_encoding(b'\xff\xfe# coding: utf-8\n--verbose'), ('utf-16-le', 2)) + + self.assertEqual(determine_file_encoding(b'# coding: utf-8\n--verbose'), ('utf-8', 0)) + self.assertEqual(determine_file_encoding(b'# coding: someencodinghere-12345\n--verbose'), ('someencodinghere-12345', 0)) + + self.assertEqual(determine_file_encoding(b'#coding:utf-8\n--verbose'), ('utf-8', 0)) + self.assertEqual(determine_file_encoding(b'# coding: utf-8 \r\n--verbose'), ('utf-8', 0)) + + self.assertEqual(determine_file_encoding('# coding: utf-32-be'.encode('utf-32-be')), ('utf-32-be', 0)) + self.assertEqual(determine_file_encoding('# coding: utf-16-le'.encode('utf-16-le')), ('utf-16-le', 0)) + + def test_get_compatible_ext(self): + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None, None], vexts=['mp4'], aexts=['m4a', 'm4a']), 'mkv') + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None], vexts=['flv'], aexts=['flv']), 'flv') + + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None], vexts=['mp4'], aexts=['m4a']), 'mp4') + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None], vexts=['mp4'], aexts=['webm']), 'mkv') + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None], vexts=['webm'], aexts=['m4a']), 'mkv') + self.assertEqual(get_compatible_ext( + vcodecs=[None], acodecs=[None], vexts=['webm'], aexts=['webm']), 'webm') + + self.assertEqual(get_compatible_ext( + vcodecs=['h264'], acodecs=['mp4a'], vexts=['mov'], aexts=['m4a']), 'mp4') + self.assertEqual(get_compatible_ext( + vcodecs=['av01.0.12M.08'], acodecs=['opus'], vexts=['mp4'], aexts=['webm']), 'webm') + + self.assertEqual(get_compatible_ext( + vcodecs=['vp9'], acodecs=['opus'], vexts=['webm'], aexts=['webm'], preferences=['flv', 'mp4']), 'mp4') + self.assertEqual(get_compatible_ext( + vcodecs=['av1'], acodecs=['mp4a'], vexts=['webm'], aexts=['m4a'], preferences=('webm', 'mkv')), 'mkv') + + def test_traverse_obj(self): + _TEST_DATA = { + 100: 100, + 1.2: 1.2, + 'str': 'str', + 'None': None, + '...': ..., + 'urls': [ + {'index': 0, 'url': 'https://www.example.com/0'}, + {'index': 1, 'url': 'https://www.example.com/1'}, + ], + 'data': ( + {'index': 2}, + {'index': 3}, + ), + 'dict': {}, + } + + # Test base functionality + self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str', + msg='allow tuple path') + self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str', + msg='allow list path') + self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str', + msg='allow iterable path') + self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str', + msg='single items should be treated as a path') + self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA) + self.assertEqual(traverse_obj(_TEST_DATA, 100), 100) + self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2) + + # Test Ellipsis behavior + self.assertCountEqual(traverse_obj(_TEST_DATA, ...), + (item for item in _TEST_DATA.values() if item is not None), + msg='`...` should give all values except `None`') + self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, ...)), _TEST_DATA['urls'][0].values(), + msg='`...` selection for dicts should select all values') + self.assertEqual(traverse_obj(_TEST_DATA, (..., ..., 'url')), + ['https://www.example.com/0', 'https://www.example.com/1'], + msg='nested `...` queries should work') + self.assertCountEqual(traverse_obj(_TEST_DATA, (..., ..., 'index')), range(4), + msg='`...` query result should be flattened') + + # Test function as key + self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)), + [_TEST_DATA['urls']], + msg='function as query key should perform a filter based on (key, value)') + self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'}, + msg='exceptions in the query function should be catched') + + # Test alternative paths + self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str', + msg='multiple `paths` should be treated as alternative paths') + self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str', + msg='alternatives should exit early') + self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None, + msg='alternatives should return `default` if exhausted') + self.assertEqual(traverse_obj(_TEST_DATA, (..., 'fail'), 100), 100, + msg='alternatives should track their own branching return') + self.assertEqual(traverse_obj(_TEST_DATA, ('dict', ...), ('data', ...)), list(_TEST_DATA['data']), + msg='alternatives on empty objects should search further') + + # Test branch and path nesting + self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'], + msg='tuple as key should be treated as branches') + self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'], + msg='list as key should be treated as branches') + self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'], + msg='double nesting in path should be treated as paths') + self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1], + msg='do not fail early on branching') + self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))), + ['https://www.example.com/0', 'https://www.example.com/1'], + msg='tripple nesting in path should be treated as branches') + self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (..., 'url')))), + ['https://www.example.com/0', 'https://www.example.com/1'], + msg='ellipsis as branch path start gets flattened') + + # Test dictionary as key + self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2}, + msg='dict key should result in a dict with the same keys') + self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}), + {0: 'https://www.example.com/0'}, + msg='dict key should allow paths') + self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}), + {0: ['https://www.example.com/0']}, + msg='tuple in dict path should be treated as branches') + self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}), + {0: ['https://www.example.com/0']}, + msg='double nesting in dict path should be treated as paths') + self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}), + {0: ['https://www.example.com/1', 'https://www.example.com/0']}, + msg='tripple nesting in dict path should be treated as branches') + self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {}, + msg='remove `None` values when dict key') + self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=...), {0: ...}, + msg='do not remove `None` values if `default`') + self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {0: {}}, + msg='do not remove empty values when dict key') + self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=...), {0: {}}, + msg='do not remove empty values when dict key and a default') + self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', ...)}), {0: []}, + msg='if branch in dict key not successful, return `[]`') + + # Testing default parameter behavior + _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []} + self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None, + msg='default value should be `None`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=...), ..., + msg='chained fails should result in default') + self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0, + msg='should not short cirquit on `None`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1, + msg='invalid dict key should result in `default`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1, + msg='`None` is a deliberate sentinel and should become `default`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None, + msg='`IndexError` should result in `default`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=1), 1, + msg='if branched but not successful return `default` if defined, not `[]`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail'), default=None), None, + msg='if branched but not successful return `default` even if `default` is `None`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, (..., 'fail')), [], + msg='if branched but not successful return `[]`, not `default`') + self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', ...)), [], + msg='if branched but object is empty return `[]`, not `default`') + + # Testing expected_type behavior + _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0} + self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str), 'str', + msg='accept matching `expected_type` type') + self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int), None, + msg='reject non matching `expected_type` type') + self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)), '0', + msg='transform type using type function') + self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', + expected_type=lambda _: 1 / 0), None, + msg='wrap expected_type fuction in try_call') + self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, ..., expected_type=str), ['str'], + msg='eliminate items that expected_type fails on') + + # Test get_all behavior + _GET_ALL_DATA = {'key': [0, 1, 2]} + self.assertEqual(traverse_obj(_GET_ALL_DATA, ('key', ...), get_all=False), 0, + msg='if not `get_all`, return only first matching value') + self.assertEqual(traverse_obj(_GET_ALL_DATA, ..., get_all=False), [0, 1, 2], + msg='do not overflatten if not `get_all`') + + # Test casesense behavior + _CASESENSE_DATA = { + 'KeY': 'value0', + 0: { + 'KeY': 'value1', + 0: {'KeY': 'value2'}, + }, + } + self.assertEqual(traverse_obj(_CASESENSE_DATA, 'key'), None, + msg='dict keys should be case sensitive unless `casesense`') + self.assertEqual(traverse_obj(_CASESENSE_DATA, 'keY', + casesense=False), 'value0', + msg='allow non matching key case if `casesense`') + self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ('keY',)), + casesense=False), ['value1'], + msg='allow non matching key case in branch if `casesense`') + self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ((0, 'keY'),)), + casesense=False), ['value2'], + msg='allow non matching key case in branch path if `casesense`') + + # Test traverse_string behavior + _TRAVERSE_STRING_DATA = {'str': 'str', 1.2: 1.2} + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0)), None, + msg='do not traverse into string if not `traverse_string`') + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0), + traverse_string=True), 's', + msg='traverse into string if `traverse_string`') + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, (1.2, 1), + traverse_string=True), '.', + msg='traverse into converted data if `traverse_string`') + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', ...), + traverse_string=True), list('str'), + msg='`...` branching into string should result in list') + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)), + traverse_string=True), ['s', 'r'], + msg='branching into string should result in list') + self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda _, x: x), + traverse_string=True), list('str'), + msg='function branching into string should result in list') + + # Test is_user_input behavior + _IS_USER_INPUT_DATA = {'range8': list(range(8))} + self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'), + is_user_input=True), 3, + msg='allow for string indexing if `is_user_input`') + self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'), + is_user_input=True), tuple(range(8))[3:], + msg='allow for string slice if `is_user_input`') + self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'), + is_user_input=True), tuple(range(8))[:4:2], + msg='allow step in string slice if `is_user_input`') + self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'), + is_user_input=True), range(8), + msg='`:` should be treated as `...` if `is_user_input`') + with self.assertRaises(TypeError, msg='too many params should result in error'): + traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), is_user_input=True) + + # Test re.Match as input obj + mobj = re.fullmatch(r'0(12)(?P<group>3)(4)?', '0123') + self.assertEqual(traverse_obj(mobj, ...), [x for x in mobj.groups() if x is not None], + msg='`...` on a `re.Match` should give its `groups()`') + self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 2)), ['0123', '3'], + msg='function on a `re.Match` should give groupno, value starting at 0') + self.assertEqual(traverse_obj(mobj, 'group'), '3', + msg='str key on a `re.Match` should give group with that name') + self.assertEqual(traverse_obj(mobj, 2), '3', + msg='int key on a `re.Match` should give group with that name') + self.assertEqual(traverse_obj(mobj, 'gRoUp', casesense=False), '3', + msg='str key on a `re.Match` should respect casesense') + self.assertEqual(traverse_obj(mobj, 'fail'), None, + msg='failing str key on a `re.Match` should return `default`') + self.assertEqual(traverse_obj(mobj, 'gRoUpS', casesense=False), None, + msg='failing str key on a `re.Match` should return `default`') + self.assertEqual(traverse_obj(mobj, 8), None, + msg='failing int key on a `re.Match` should return `default`') + if __name__ == '__main__': unittest.main() diff --git a/test/test_verbose_output.py b/test/test_verbose_output.py index 98c6d70..de0cb8e 100644 --- a/test/test_verbose_output.py +++ b/test/test_verbose_output.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 -# coding: utf-8 - -from __future__ import unicode_literals +# Allow direct execution +import os +import sys import unittest -import sys -import os -import subprocess sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import subprocess + rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -17,7 +17,8 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_arg(self): outp = subprocess.Popen( [ - sys.executable, 'hypervideo_dl/__main__.py', '-v', + sys.executable, 'hypervideo_dl/__main__.py', + '-v', '--ignore-config', '--username', 'johnsmith@gmail.com', '--password', 'my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -30,7 +31,8 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_shortarg(self): outp = subprocess.Popen( [ - sys.executable, 'hypervideo_dl/__main__.py', '-v', + sys.executable, 'hypervideo_dl/__main__.py', + '-v', '--ignore-config', '-u', 'johnsmith@gmail.com', '-p', 'my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -43,7 +45,8 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_eq(self): outp = subprocess.Popen( [ - sys.executable, 'hypervideo_dl/__main__.py', '-v', + sys.executable, 'hypervideo_dl/__main__.py', + '-v', '--ignore-config', '--username=johnsmith@gmail.com', '--password=my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -56,7 +59,8 @@ class TestVerboseOutput(unittest.TestCase): def test_private_info_shortarg_eq(self): outp = subprocess.Popen( [ - sys.executable, 'hypervideo_dl/__main__.py', '-v', + sys.executable, 'hypervideo_dl/__main__.py', + '-v', '--ignore-config', '-u=johnsmith@gmail.com', '-p=my_secret_password', ], cwd=rootDir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py deleted file mode 100644 index 6f6c7ab..0000000 --- a/test/test_write_annotations.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -from __future__ import unicode_literals - -# Allow direct execution -import os -import sys -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from test.helper import get_params, try_rm - - -import io - -import xml.etree.ElementTree - -import hypervideo_dl.YoutubeDL -import hypervideo_dl.extractor - - -class YoutubeDL(hypervideo_dl.YoutubeDL): - def __init__(self, *args, **kwargs): - super(YoutubeDL, self).__init__(*args, **kwargs) - self.to_stderr = self.to_screen - - -params = get_params({ - 'writeannotations': True, - 'skip_download': True, - 'writeinfojson': False, - 'format': 'flv', -}) - - -TEST_ID = 'gr51aVj-mLg' -ANNOTATIONS_FILE = TEST_ID + '.annotations.xml' -EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label'] - - -class TestAnnotations(unittest.TestCase): - def setUp(self): - # Clear old files - self.tearDown() - - def test_info_json(self): - expected = list(EXPECTED_ANNOTATIONS) # Two annotations could have the same text. - ie = hypervideo_dl.extractor.YoutubeIE() - ydl = YoutubeDL(params) - ydl.add_info_extractor(ie) - ydl.download([TEST_ID]) - self.assertTrue(os.path.exists(ANNOTATIONS_FILE)) - annoxml = None - with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof: - annoxml = xml.etree.ElementTree.parse(annof) - self.assertTrue(annoxml is not None, 'Failed to parse annotations XML') - root = annoxml.getroot() - self.assertEqual(root.tag, 'document') - annotationsTag = root.find('annotations') - self.assertEqual(annotationsTag.tag, 'annotations') - annotations = annotationsTag.findall('annotation') - - # Not all the annotations have TEXT children and the annotations are returned unsorted. - for a in annotations: - self.assertEqual(a.tag, 'annotation') - if a.get('type') == 'text': - textTag = a.find('TEXT') - text = textTag.text - self.assertTrue(text in expected) # assertIn only added in python 2.7 - # remove the first occurrence, there could be more than one annotation with the same text - expected.remove(text) - # We should have seen (and removed) all the expected annotation texts. - self.assertEqual(len(expected), 0, 'Not all expected annotations were found.') - - def tearDown(self): - try_rm(ANNOTATIONS_FILE) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index b94b733..50f42d1 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -1,18 +1,16 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, is_download_test -from hypervideo_dl.extractor import ( - YoutubeIE, - YoutubeTabIE, -) +from test.helper import FakeYDL, is_download_test +from hypervideo_dl.extractor import YoutubeIE, YoutubeTabIE +from hypervideo_dl.utils import ExtractorError @is_download_test @@ -56,6 +54,18 @@ class TestYoutubeLists(unittest.TestCase): self.assertEqual(video['duration'], 10) self.assertEqual(video['uploader'], 'Philipp Hagemeister') + def test_youtube_channel_no_uploads(self): + dl = FakeYDL() + dl.params['extract_flat'] = True + ie = YoutubeTabIE(dl) + # no uploads + with self.assertRaisesRegex(ExtractorError, r'no uploads'): + ie.extract('https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA') + + # no uploads and no UCID given + with self.assertRaisesRegex(ExtractorError, r'no uploads'): + ie.extract('https://www.youtube.com/news') + if __name__ == '__main__': unittest.main() diff --git a/test/test_youtube_misc.py b/test/test_youtube_misc.py index 4571cc1..2889795 100644 --- a/test/test_youtube_misc.py +++ b/test/test_youtube_misc.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals # Allow direct execution import os import sys import unittest + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/test/testdata/certificate/ca.crt b/test/testdata/certificate/ca.crt new file mode 100644 index 0000000..ddf7be7 --- /dev/null +++ b/test/testdata/certificate/ca.crt @@ -0,0 +1,10 @@ +-----BEGIN CERTIFICATE----- +MIIBfDCCASOgAwIBAgIUUgngoxFpuWft8gjj3uEFoqJyoJowCgYIKoZIzj0EAwIw +FDESMBAGA1UEAwwJeXRkbHB0ZXN0MB4XDTIyMDQxNTAzMDEwMVoXDTM4MTAxNTAz +MDEwMVowFDESMBAGA1UEAwwJeXRkbHB0ZXN0MFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAEcTaKMtIn2/1kgid1zXFpLm87FMT5PP3/bltKVVH3DLO//0kUslCHYxFU +KpcCfVt9aueRyUFi1TNkkkEZ9D6fbqNTMFEwHQYDVR0OBBYEFBdY2rVNLFGM6r1F +iuamNDaiq0QoMB8GA1UdIwQYMBaAFBdY2rVNLFGM6r1FiuamNDaiq0QoMA8GA1Ud +EwEB/wQFMAMBAf8wCgYIKoZIzj0EAwIDRwAwRAIgXJg2jio1kow2g/iP54Qq+iI2 +m4EAvZiY0Im/Ni3PHawCIC6KCl6QcHANbeq8ckOXNGusjl6OWhvEM3uPBPhqskq1 +-----END CERTIFICATE----- diff --git a/test/testdata/certificate/ca.key b/test/testdata/certificate/ca.key new file mode 100644 index 0000000..38920d5 --- /dev/null +++ b/test/testdata/certificate/ca.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIG2L1bHdl3PnaLiJ7Zm8aAGCj4GiVbSbXQcrJAdL+yqOoAoGCCqGSM49 +AwEHoUQDQgAEcTaKMtIn2/1kgid1zXFpLm87FMT5PP3/bltKVVH3DLO//0kUslCH +YxFUKpcCfVt9aueRyUFi1TNkkkEZ9D6fbg== +-----END EC PRIVATE KEY----- diff --git a/test/testdata/certificate/ca.srl b/test/testdata/certificate/ca.srl new file mode 100644 index 0000000..de2d1ea --- /dev/null +++ b/test/testdata/certificate/ca.srl @@ -0,0 +1 @@ +4A260C33C4D34612646E6321E1E767DF1A95EF0B diff --git a/test/testdata/certificate/client.crt b/test/testdata/certificate/client.crt new file mode 100644 index 0000000..874622f --- /dev/null +++ b/test/testdata/certificate/client.crt @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBIzCBygIUSiYMM8TTRhJkbmMh4edn3xqV7wswCgYIKoZIzj0EAwIwFDESMBAG +A1UEAwwJeXRkbHB0ZXN0MB4XDTIyMDQxNTAzMDEyN1oXDTM4MTAxNTAzMDEyN1ow +FTETMBEGA1UEAwwKeXRkbHB0ZXN0MjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BKREKVDWfLKZknzYg+BUkmTn43f2pl/LNSyKPtXo/UV7hhp6JXIq3ZuZ7rubyuMS +XNuH+2Cl9msSpJB2LhJs5kcwCgYIKoZIzj0EAwIDSAAwRQIhAMRr46vO25/5nUhD +aHp4L67AeSvrjvSFHfubyD3Kr5dwAiA8EfOgVxc8Qh6ozTcbXO/WnBfS48ZFRSQY +D0dB8M1kJw== +-----END CERTIFICATE----- diff --git a/test/testdata/certificate/client.csr b/test/testdata/certificate/client.csr new file mode 100644 index 0000000..2d5d7a5 --- /dev/null +++ b/test/testdata/certificate/client.csr @@ -0,0 +1,7 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIHQMHcCAQAwFTETMBEGA1UEAwwKeXRkbHB0ZXN0MjBZMBMGByqGSM49AgEGCCqG +SM49AwEHA0IABKREKVDWfLKZknzYg+BUkmTn43f2pl/LNSyKPtXo/UV7hhp6JXIq +3ZuZ7rubyuMSXNuH+2Cl9msSpJB2LhJs5kegADAKBggqhkjOPQQDAgNJADBGAiEA +1LZ72mtPmVxhGtdMvpZ0fyA68H2RC5IMHpLq18T55UcCIQDKpkXXVTvAzS0JioCq +6kiYq8Oxx6ZMoI+11k75/Kip1g== +-----END CERTIFICATE REQUEST----- diff --git a/test/testdata/certificate/client.key b/test/testdata/certificate/client.key new file mode 100644 index 0000000..e47389b --- /dev/null +++ b/test/testdata/certificate/client.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIAW6h9hwT0Aha+JBukgmHnrKRPoqPNWYA86ic0UaKHs8oAoGCCqGSM49 +AwEHoUQDQgAEpEQpUNZ8spmSfNiD4FSSZOfjd/amX8s1LIo+1ej9RXuGGnolcird +m5nuu5vK4xJc24f7YKX2axKkkHYuEmzmRw== +-----END EC PRIVATE KEY----- diff --git a/test/testdata/certificate/clientencrypted.key b/test/testdata/certificate/clientencrypted.key new file mode 100644 index 0000000..0baee37 --- /dev/null +++ b/test/testdata/certificate/clientencrypted.key @@ -0,0 +1,8 @@ +-----BEGIN EC PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,4B39160146F15544922E553E08299A35 + +96A7/iBkIfTVb8r2812ued2pS49FfVY4Ppz/45OGF0uFayMtMl8/GuEBCamuhFXS +rnOOpco96TTeeKZHqR45wnf4tgHM8IjoQ6H0EX3lVF19OHnArAgrGYtohWUGSyGn +IgLJFdUewIjdI7XApTJprQFE5E2tETXFA95mCz88u1c= +-----END EC PRIVATE KEY----- diff --git a/test/testdata/certificate/clientwithencryptedkey.crt b/test/testdata/certificate/clientwithencryptedkey.crt new file mode 100644 index 0000000..f357e4c --- /dev/null +++ b/test/testdata/certificate/clientwithencryptedkey.crt @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIIBIzCBygIUSiYMM8TTRhJkbmMh4edn3xqV7wswCgYIKoZIzj0EAwIwFDESMBAG +A1UEAwwJeXRkbHB0ZXN0MB4XDTIyMDQxNTAzMDEyN1oXDTM4MTAxNTAzMDEyN1ow +FTETMBEGA1UEAwwKeXRkbHB0ZXN0MjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BKREKVDWfLKZknzYg+BUkmTn43f2pl/LNSyKPtXo/UV7hhp6JXIq3ZuZ7rubyuMS +XNuH+2Cl9msSpJB2LhJs5kcwCgYIKoZIzj0EAwIDSAAwRQIhAMRr46vO25/5nUhD +aHp4L67AeSvrjvSFHfubyD3Kr5dwAiA8EfOgVxc8Qh6ozTcbXO/WnBfS48ZFRSQY +D0dB8M1kJw== +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,4B39160146F15544922E553E08299A35 + +96A7/iBkIfTVb8r2812ued2pS49FfVY4Ppz/45OGF0uFayMtMl8/GuEBCamuhFXS +rnOOpco96TTeeKZHqR45wnf4tgHM8IjoQ6H0EX3lVF19OHnArAgrGYtohWUGSyGn +IgLJFdUewIjdI7XApTJprQFE5E2tETXFA95mCz88u1c= +-----END EC PRIVATE KEY----- diff --git a/test/testdata/certificate/clientwithkey.crt b/test/testdata/certificate/clientwithkey.crt new file mode 100644 index 0000000..942f6e2 --- /dev/null +++ b/test/testdata/certificate/clientwithkey.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIIBIzCBygIUSiYMM8TTRhJkbmMh4edn3xqV7wswCgYIKoZIzj0EAwIwFDESMBAG +A1UEAwwJeXRkbHB0ZXN0MB4XDTIyMDQxNTAzMDEyN1oXDTM4MTAxNTAzMDEyN1ow +FTETMBEGA1UEAwwKeXRkbHB0ZXN0MjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA +BKREKVDWfLKZknzYg+BUkmTn43f2pl/LNSyKPtXo/UV7hhp6JXIq3ZuZ7rubyuMS +XNuH+2Cl9msSpJB2LhJs5kcwCgYIKoZIzj0EAwIDSAAwRQIhAMRr46vO25/5nUhD +aHp4L67AeSvrjvSFHfubyD3Kr5dwAiA8EfOgVxc8Qh6ozTcbXO/WnBfS48ZFRSQY +D0dB8M1kJw== +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIAW6h9hwT0Aha+JBukgmHnrKRPoqPNWYA86ic0UaKHs8oAoGCCqGSM49 +AwEHoUQDQgAEpEQpUNZ8spmSfNiD4FSSZOfjd/amX8s1LIo+1ej9RXuGGnolcird +m5nuu5vK4xJc24f7YKX2axKkkHYuEmzmRw== +-----END EC PRIVATE KEY----- diff --git a/test/testdata/certificate/instructions.md b/test/testdata/certificate/instructions.md new file mode 100644 index 0000000..b0e3fbd --- /dev/null +++ b/test/testdata/certificate/instructions.md @@ -0,0 +1,19 @@ +# Generate certificates for client cert tests + +## CA +```sh +openssl ecparam -name prime256v1 -genkey -noout -out ca.key +openssl req -new -x509 -sha256 -days 6027 -key ca.key -out ca.crt -subj "/CN=ytdlptest" +``` + +## Client +```sh +openssl ecparam -name prime256v1 -genkey -noout -out client.key +openssl ec -in client.key -out clientencrypted.key -passout pass:foobar -aes256 +openssl req -new -sha256 -key client.key -out client.csr -subj "/CN=ytdlptest2" +openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 6027 -sha256 +cp client.crt clientwithkey.crt +cp client.crt clientwithencryptedkey.crt +cat client.key >> clientwithkey.crt +cat clientencrypted.key >> clientwithencryptedkey.crt +```
\ No newline at end of file diff --git a/test/testdata/ism/ec-3_test.Manifest b/test/testdata/ism/ec-3_test.Manifest new file mode 100644 index 0000000..45f95de --- /dev/null +++ b/test/testdata/ism/ec-3_test.Manifest @@ -0,0 +1 @@ +<?xml version="1.0" encoding="utf-8"?><!--Transformed by VSMT using XSL stylesheet for rule Identity--><!-- Created with Unified Streaming Platform (version=1.10.12-18737) --><SmoothStreamingMedia MajorVersion="2" MinorVersion="0" TimeScale="10000000" Duration="370000000"><StreamIndex Type="audio" QualityLevels="1" TimeScale="10000000" Language="deu" Name="audio_deu" Chunks="19" Url="QualityLevels({bitrate})/Fragments(audio_deu={start time})?noStreamProfile=1"><QualityLevel Index="0" Bitrate="127802" CodecPrivateData="1190" SamplingRate="48000" Channels="2" BitsPerSample="16" PacketSize="4" AudioTag="255" FourCC="AACL" /><c t="0" d="20053333" /><c d="20053334" /><c d="20053333" /><c d="19840000" /><c d="20053333" /><c d="20053334" /><c d="20053333" /><c d="19840000" /><c d="20053333" /><c d="20053334" /><c d="20053333" /><c d="19840000" /><c d="20053333" /><c d="20053334" /><c d="20053333" /><c d="19840000" /><c d="20053333" /><c d="20053334" /><c d="7253333" /></StreamIndex><StreamIndex Type="audio" QualityLevels="1" TimeScale="10000000" Language="deu" Name="audio_deu_1" Chunks="19" Url="QualityLevels({bitrate})/Fragments(audio_deu_1={start time})?noStreamProfile=1"><QualityLevel Index="0" Bitrate="224000" CodecPrivateData="00063F000000AF87FBA7022DFB42A4D405CD93843BDD0700200F00" FourCCData="0700200F00" SamplingRate="48000" Channels="6" BitsPerSample="16" PacketSize="896" AudioTag="65534" FourCC="EC-3" /><c t="0" d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="20160000" /><c d="19840000" /><c d="8320000" /></StreamIndex><StreamIndex Type="video" QualityLevels="8" TimeScale="10000000" Language="deu" Name="video_deu" Chunks="19" Url="QualityLevels({bitrate})/Fragments(video_deu={start time})?noStreamProfile=1" MaxWidth="1920" MaxHeight="1080" DisplayWidth="1920" DisplayHeight="1080"><QualityLevel Index="0" Bitrate="23909" CodecPrivateData="000000016742C00CDB06077E5C05A808080A00000300020000030009C0C02EE0177CC6300F142AE00000000168CA8DC8" MaxWidth="384" MaxHeight="216" FourCC="AVC1" /><QualityLevel Index="1" Bitrate="403188" CodecPrivateData="00000001674D4014E98323B602D4040405000003000100000300320F1429380000000168EAECF2" MaxWidth="400" MaxHeight="224" FourCC="AVC1" /><QualityLevel Index="2" Bitrate="680365" CodecPrivateData="00000001674D401EE981405FF2E02D4040405000000300100000030320F162D3800000000168EAECF2" MaxWidth="640" MaxHeight="360" FourCC="AVC1" /><QualityLevel Index="3" Bitrate="1253465" CodecPrivateData="00000001674D401EE981405FF2E02D4040405000000300100000030320F162D3800000000168EAECF2" MaxWidth="640" MaxHeight="360" FourCC="AVC1" /><QualityLevel Index="4" Bitrate="2121558" CodecPrivateData="00000001674D401EECA0601BD80B50101014000003000400000300C83C58B6580000000168E93B3C80" MaxWidth="768" MaxHeight="432" FourCC="AVC1" /><QualityLevel Index="5" Bitrate="3275545" CodecPrivateData="00000001674D4020ECA02802DD80B501010140000003004000000C83C60C65800000000168E93B3C80" MaxWidth="1280" MaxHeight="720" FourCC="AVC1" /><QualityLevel Index="6" Bitrate="5300196" CodecPrivateData="00000001674D4028ECA03C0113F2E02D4040405000000300100000030320F18319600000000168E93B3C80" MaxWidth="1920" MaxHeight="1080" FourCC="AVC1" /><QualityLevel Index="7" Bitrate="8079312" CodecPrivateData="00000001674D4028ECA03C0113F2E02D4040405000000300100000030320F18319600000000168E93B3C80" MaxWidth="1920" MaxHeight="1080" FourCC="AVC1" /><c t="0" d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="20000000" /><c d="10000000" /></StreamIndex></SmoothStreamingMedia>
\ No newline at end of file diff --git a/test/testdata/m3u8/pluzz_francetv_11507.m3u8 b/test/testdata/m3u8/pluzz_francetv_11507.m3u8 deleted file mode 100644 index 0809f5a..0000000 --- a/test/testdata/m3u8/pluzz_francetv_11507.m3u8 +++ /dev/null @@ -1,14 +0,0 @@ -#EXTM3U -
#EXT-X-VERSION:5 -
#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Francais",DEFAULT=NO,FORCED=NO,URI="http://replayftv-pmd.francetv.fr/subtitles/2017/16/156589847-1492488987.m3u8",LANGUAGE="fra" -
#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aac",LANGUAGE="fra",NAME="Francais",DEFAULT=YES, AUTOSELECT=YES -#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=180000,RESOLUTION=256x144,CODECS="avc1.66.30, mp4a.40.2" -http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_0_av.m3u8?null=0 -#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=303000,RESOLUTION=320x180,CODECS="avc1.66.30, mp4a.40.2" -http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_1_av.m3u8?null=0 -#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=575000,RESOLUTION=512x288,CODECS="avc1.66.30, mp4a.40.2" -http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_2_av.m3u8?null=0 -#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=831000,RESOLUTION=704x396,CODECS="avc1.77.30, mp4a.40.2" -http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_3_av.m3u8?null=0 -#EXT-X-STREAM-INF:SUBTITLES="subs",AUDIO="aac",PROGRAM-ID=1,BANDWIDTH=1467000,RESOLUTION=1024x576,CODECS="avc1.77.30, mp4a.40.2" -http://replayftv-vh.akamaihd.net/i/streaming-adaptatif_france-dom-tom/2017/S16/J2/156589847-58f59130c1f52-,standard1,standard2,standard3,standard4,standard5,.mp4.csmil/index_4_av.m3u8?null=0 diff --git a/test/testdata/m3u8/teamcoco_11995.m3u8 b/test/testdata/m3u8/teamcoco_11995.m3u8 deleted file mode 100644 index a6e4216..0000000 --- a/test/testdata/m3u8/teamcoco_11995.m3u8 +++ /dev/null @@ -1,16 +0,0 @@ -#EXTM3U -#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-0",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-160k_v4.m3u8" -#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio-1",NAME="Default",AUTOSELECT=YES,DEFAULT=YES,URI="hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8" -#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=37862000,CODECS="avc1.4d001f",URI="hls/CONAN_020217_Highlight_show-2m_iframe.m3u8" -#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=18750000,CODECS="avc1.4d001e",URI="hls/CONAN_020217_Highlight_show-1m_iframe.m3u8" -#EXT-X-I-FRAME-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=6535000,CODECS="avc1.42001e",URI="hls/CONAN_020217_Highlight_show-400k_iframe.m3u8" -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=2374000,RESOLUTION=1024x576,CODECS="avc1.4d001f,mp4a.40.2",AUDIO="audio-0" -hls/CONAN_020217_Highlight_show-2m_v4.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1205000,RESOLUTION=640x360,CODECS="avc1.4d001e,mp4a.40.2",AUDIO="audio-0" -hls/CONAN_020217_Highlight_show-1m_v4.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=522000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.2",AUDIO="audio-0" -hls/CONAN_020217_Highlight_show-400k_v4.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=413000,RESOLUTION=400x224,CODECS="avc1.42001e,mp4a.40.5",AUDIO="audio-1" -hls/CONAN_020217_Highlight_show-400k_v4.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=71000,CODECS="mp4a.40.5",AUDIO="audio-1" -hls/CONAN_020217_Highlight_show-audio-64k_v4.m3u8 diff --git a/test/testdata/m3u8/ted_18923.m3u8 b/test/testdata/m3u8/ted_18923.m3u8 deleted file mode 100644 index 52a2711..0000000 --- a/test/testdata/m3u8/ted_18923.m3u8 +++ /dev/null @@ -1,28 +0,0 @@ -#EXTM3U -#EXT-X-VERSION:4 -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=1255659,PROGRAM-ID=1,CODECS="avc1.42c01e,mp4a.40.2",RESOLUTION=640x360 -/videos/BorisHesser_2018S/video/600k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=163154,PROGRAM-ID=1,CODECS="avc1.42c00c,mp4a.40.2",RESOLUTION=320x180 -/videos/BorisHesser_2018S/video/64k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=481701,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288 -/videos/BorisHesser_2018S/video/180k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=769968,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288 -/videos/BorisHesser_2018S/video/320k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=984037,PROGRAM-ID=1,CODECS="avc1.42c015,mp4a.40.2",RESOLUTION=512x288 -/videos/BorisHesser_2018S/video/450k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=1693925,PROGRAM-ID=1,CODECS="avc1.4d401f,mp4a.40.2",RESOLUTION=853x480 -/videos/BorisHesser_2018S/video/950k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=2462469,PROGRAM-ID=1,CODECS="avc1.640028,mp4a.40.2",RESOLUTION=1280x720 -/videos/BorisHesser_2018S/video/1500k.m3u8?nobumpers=true&uniqueId=76011e2b -#EXT-X-STREAM-INF:AUDIO="600k",BANDWIDTH=68101,PROGRAM-ID=1,CODECS="mp4a.40.2",DEFAULT=YES -/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b - -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=74298,PROGRAM-ID=1,CODECS="avc1.42c00c",RESOLUTION=320x180,URI="/videos/BorisHesser_2018S/video/64k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=216200,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/180k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=304717,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/320k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=350933,PROGRAM-ID=1,CODECS="avc1.42c015",RESOLUTION=512x288,URI="/videos/BorisHesser_2018S/video/450k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=495850,PROGRAM-ID=1,CODECS="avc1.42c01e",RESOLUTION=640x360,URI="/videos/BorisHesser_2018S/video/600k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=810750,PROGRAM-ID=1,CODECS="avc1.4d401f",RESOLUTION=853x480,URI="/videos/BorisHesser_2018S/video/950k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" -#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=1273700,PROGRAM-ID=1,CODECS="avc1.640028",RESOLUTION=1280x720,URI="/videos/BorisHesser_2018S/video/1500k_iframe.m3u8?nobumpers=true&uniqueId=76011e2b" - -#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="600k",LANGUAGE="en",NAME="Audio",AUTOSELECT=YES,DEFAULT=YES,URI="/videos/BorisHesser_2018S/audio/600k.m3u8?nobumpers=true&uniqueId=76011e2b",BANDWIDTH=614400 diff --git a/test/testdata/m3u8/toggle_mobile_12211.m3u8 b/test/testdata/m3u8/toggle_mobile_12211.m3u8 deleted file mode 100644 index 69604e6..0000000 --- a/test/testdata/m3u8/toggle_mobile_12211.m3u8 +++ /dev/null @@ -1,13 +0,0 @@ -#EXTM3U -#EXT-X-VERSION:4 -#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="eng",NAME="English",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_sa2ntrdg/name/a.mp4/index.m3u8" -#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",LANGUAGE="und",NAME="Undefined",URI="http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_r7y0nitg/name/a.mp4/index.m3u8" - -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=155648,RESOLUTION=320x180,AUDIO="audio" -http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_qlk9hlzr/name/a.mp4/index.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=502784,RESOLUTION=480x270,AUDIO="audio" -http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/2/pv/1/flavorId/0_oefackmi/name/a.mp4/index.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=827392,RESOLUTION=640x360,AUDIO="audio" -http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_vyg9pj7k/name/a.mp4/index.m3u8 -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1396736,RESOLUTION=854x480,AUDIO="audio" -http://k.toggle.sg/fhls/p/2082311/sp/208231100/serveFlavor/entryId/0_89q6e8ku/v/12/pv/1/flavorId/0_50n4psvx/name/a.mp4/index.m3u8 diff --git a/test/testdata/m3u8/twitch_vod.m3u8 b/test/testdata/m3u8/twitch_vod.m3u8 deleted file mode 100644 index 7617277..0000000 --- a/test/testdata/m3u8/twitch_vod.m3u8 +++ /dev/null @@ -1,20 +0,0 @@ -#EXTM3U -#EXT-X-TWITCH-INFO:ORIGIN="s3",CLUSTER="edgecast_vod",REGION="EU",MANIFEST-CLUSTER="edgecast_vod",USER-IP="109.171.17.81" -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="chunked",NAME="Source",AUTOSELECT=YES,DEFAULT=YES -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=3214134,CODECS="avc1.100.31,mp4a.40.2",RESOLUTION="1280x720",VIDEO="chunked" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/chunked/index-muted-HM49I092CC.m3u8 -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="high",NAME="High",AUTOSELECT=YES,DEFAULT=YES -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1603789,CODECS="avc1.42C01F,mp4a.40.2",RESOLUTION="1280x720",VIDEO="high" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/high/index-muted-HM49I092CC.m3u8 -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="medium",NAME="Medium",AUTOSELECT=YES,DEFAULT=YES -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=893387,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="852x480",VIDEO="medium" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/medium/index-muted-HM49I092CC.m3u8 -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="low",NAME="Low",AUTOSELECT=YES,DEFAULT=YES -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=628347,CODECS="avc1.42C01E,mp4a.40.2",RESOLUTION="640x360",VIDEO="low" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/low/index-muted-HM49I092CC.m3u8 -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="mobile",NAME="Mobile",AUTOSELECT=YES,DEFAULT=YES -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=280474,CODECS="avc1.42C00D,mp4a.40.2",RESOLUTION="400x226",VIDEO="mobile" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/mobile/index-muted-HM49I092CC.m3u8 -#EXT-X-MEDIA:TYPE=VIDEO,GROUP-ID="audio_only",NAME="Audio Only",AUTOSELECT=NO,DEFAULT=NO -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=182725,CODECS="mp4a.40.2",VIDEO="audio_only" -https://vod.edgecast.hls.ttvnw.net/e5da31ab49_riotgames_15001215120_261543898/audio_only/index-muted-HM49I092CC.m3u8 diff --git a/test/testdata/m3u8/vidio.m3u8 b/test/testdata/m3u8/vidio.m3u8 deleted file mode 100644 index 89c2444..0000000 --- a/test/testdata/m3u8/vidio.m3u8 +++ /dev/null @@ -1,10 +0,0 @@ -#EXTM3U - -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=300000,RESOLUTION=480x270,NAME="270p 3G" -https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b300.mp4.m3u8 - -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=600000,RESOLUTION=640x360,NAME="360p SD" -https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b600.mp4.m3u8 - -#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=1200000,RESOLUTION=1280x720,NAME="720p HD" -https://cdn1-a.production.vidio.static6.com/uploads/165683/dj_ambred-4383-b1200.mp4.m3u8 |