From cefecac12cd3c70f9c7a30992c60b05c2eb5d34e Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 2 Sep 2020 20:25:25 +0200 Subject: [skip travis] renaming to avoid using same folder when using pip install for example --- test/test_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 0896f4150..663a34e07 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -15,7 +15,7 @@ import io import json import xml.etree.ElementTree -from youtube_dl.utils import ( +from youtube_dlc.utils import ( age_restricted, args_to_str, encode_base_n, @@ -105,7 +105,7 @@ from youtube_dl.utils import ( cli_bool_option, parse_codecs, ) -from youtube_dl.compat import ( +from youtube_dlc.compat import ( compat_chr, compat_etree_fromstring, compat_getenv, @@ -240,12 +240,12 @@ class TestUtil(unittest.TestCase): def env(var): return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var) - compat_setenv('YOUTUBE_DL_EXPATH_PATH', 'expanded') - self.assertEqual(expand_path(env('YOUTUBE_DL_EXPATH_PATH')), 'expanded') + compat_setenv('youtube_dlc_EXPATH_PATH', 'expanded') + self.assertEqual(expand_path(env('youtube_dlc_EXPATH_PATH')), 'expanded') self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME')) self.assertEqual(expand_path('~'), compat_getenv('HOME')) self.assertEqual( - expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')), + expand_path('~/%s' % env('youtube_dlc_EXPATH_PATH')), '%s/expanded' % compat_getenv('HOME')) def test_prepend_extension(self): @@ -1388,8 +1388,8 @@ Line 1 self.assertEqual(caesar('ebg', 'acegik', -2), 'abc') def test_rot47(self): - self.assertEqual(rot47('youtube-dl'), r'J@FEF36\5=') - self.assertEqual(rot47('YOUTUBE-DL'), r'*~&%&qt\s{') + self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=') + self.assertEqual(rot47('youtube-dlc'), r'*~&%&qt\s{') def test_urshift(self): self.assertEqual(urshift(3, 1), 1) -- cgit v1.2.3 From 3867038a06bbd17d7d832d93495280b52c2f84f2 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 2 Sep 2020 22:37:35 +0200 Subject: renaming issues resolved --- test/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 663a34e07..80c06db7e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1388,8 +1388,8 @@ Line 1 self.assertEqual(caesar('ebg', 'acegik', -2), 'abc') def test_rot47(self): - self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=') - self.assertEqual(rot47('youtube-dlc'), r'*~&%&qt\s{') + self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=4') + self.assertEqual(rot47('youtube-dlc'), r'*~&%&qt\s{r') def test_urshift(self): self.assertEqual(urshift(3, 1), 1) -- cgit v1.2.3 From 8ef153ee6ff58ae3c2075d50a4a132e9ecd898b1 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 2 Sep 2020 22:57:46 +0200 Subject: rot47 capital letters. --- test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 80c06db7e..5914d4fd6 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1389,7 +1389,7 @@ Line 1 def test_rot47(self): self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=4') - self.assertEqual(rot47('youtube-dlc'), r'*~&%&qt\s{r') + self.assertEqual(rot47('YOUTUBE-DLC'), r'*~&%&qt\s{r') def test_urshift(self): self.assertEqual(urshift(3, 1), 1) -- cgit v1.2.3 From 39e7107d7f4553d48e0172724729b0a8ef73d7ed Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 12 Sep 2020 05:08:57 +0200 Subject: Merge branch 'ytdl-org-master' --- test/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 5914d4fd6..95231200b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -803,6 +803,8 @@ class TestUtil(unittest.TestCase): self.assertEqual(mimetype2ext('text/vtt'), 'vtt') self.assertEqual(mimetype2ext('text/vtt;charset=utf-8'), 'vtt') self.assertEqual(mimetype2ext('text/html; charset=utf-8'), 'html') + self.assertEqual(mimetype2ext('audio/x-wav'), 'wav') + self.assertEqual(mimetype2ext('audio/x-wav;codec=pcm'), 'wav') def test_month_by_name(self): self.assertEqual(month_by_name(None), None) -- cgit v1.2.3 From 8bdd16b4993b8d546b4cbbdbe4710db0bc2f971b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 20 Nov 2020 00:52:59 +0530 Subject: Merge 'ytdl-org/youtube-dl/master' release 2020.11.19 Old Extractors left behind: VLivePlaylistIE YoutubeSearchURLIE YoutubeShowIE YoutubeFavouritesIE If removing old extractors, make corresponding changes in docs/supportedsites.md youtube_dlc/extractor/extractors.py Not merged: .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md test/test_all_urls.py youtube_dlc/version.py Changelog --- test/test_utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 95231200b..16ad40831 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -937,6 +937,28 @@ class TestUtil(unittest.TestCase): self.assertEqual(d['x'], 1) self.assertEqual(d['y'], 'a') + # Just drop ! prefix for now though this results in a wrong value + on = js_to_json('''{ + a: !0, + b: !1, + c: !!0, + d: !!42.42, + e: !!![], + f: !"abc", + g: !"", + !42: 42 + }''') + self.assertEqual(json.loads(on), { + 'a': 0, + 'b': 1, + 'c': 0, + 'd': 42.42, + 'e': [], + 'f': "abc", + 'g': "", + '42': 42 + }) + on = js_to_json('["abc", "def",]') self.assertEqual(json.loads(on), ['abc', 'def']) @@ -994,6 +1016,12 @@ class TestUtil(unittest.TestCase): on = js_to_json('{42:4.2e1}') self.assertEqual(json.loads(on), {'42': 42.0}) + on = js_to_json('{ "0x40": "0x40" }') + self.assertEqual(json.loads(on), {'0x40': '0x40'}) + + on = js_to_json('{ "040": "040" }') + self.assertEqual(json.loads(on), {'040': '040'}) + def test_js_to_json_malformed(self): self.assertEqual(js_to_json('42a1'), '42"a1"') self.assertEqual(js_to_json('42a-1'), '42"a"-1') -- cgit v1.2.3 From 732044afb2e8ffbaa37fe91310906ff549edd6ad Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 27 Oct 2020 16:07:21 +0530 Subject: Add --write-*-link by h-h-h-h Authored-by: h-h-h-h --- test/test_utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 16ad40831..6562d443a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -104,6 +104,7 @@ from youtube_dlc.utils import ( cli_valueless_option, cli_bool_option, parse_codecs, + iri_to_uri, ) from youtube_dlc.compat import ( compat_chr, @@ -1465,6 +1466,32 @@ Line 1 self.assertEqual(get_elements_by_attribute('class', 'foo', html), []) self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), []) + def test_iri_to_uri(self): + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'), + 'https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b') # Same + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=Käsesoßenrührlöffel'), # German for cheese sauce stirring spoon + 'https://www.google.com/search?q=K%C3%A4seso%C3%9Fenr%C3%BChrl%C3%B6ffel') + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=lt<+gt>+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#'), + 'https://www.google.com/search?q=lt%3C+gt%3E+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#') + self.assertEqual( + iri_to_uri('http://правозащита38.рф/category/news/'), + 'http://xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + self.assertEqual( + iri_to_uri('http://www.правозащита38.рф/category/news/'), + 'http://www.xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + self.assertEqual( + iri_to_uri('https://i❤.ws/emojidomain/👍👏🤝💪'), + 'https://xn--i-7iq.ws/emojidomain/%F0%9F%91%8D%F0%9F%91%8F%F0%9F%A4%9D%F0%9F%92%AA') + self.assertEqual( + iri_to_uri('http://日本語.jp/'), + 'http://xn--wgv71a119e.jp/') + self.assertEqual( + iri_to_uri('http://导航.中国/'), + 'http://xn--fet810g.xn--fiqs8s/') + if __name__ == '__main__': unittest.main() -- cgit v1.2.3 From 29f7c58aafb25a094e267a8a3fb355e102e42792 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 1 Jan 2021 17:56:37 +0530 Subject: Update to ytdl-2021.01.03 --- test/test_utils.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 6562d443a..bb69b0522 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -555,6 +555,11 @@ class TestUtil(unittest.TestCase): self.assertEqual(url_or_none('http$://foo.de'), None) self.assertEqual(url_or_none('http://foo.de'), 'http://foo.de') self.assertEqual(url_or_none('//foo.de'), '//foo.de') + self.assertEqual(url_or_none('s3://foo.de'), None) + self.assertEqual(url_or_none('rtmpte://foo.de'), 'rtmpte://foo.de') + self.assertEqual(url_or_none('mms://foo.de'), 'mms://foo.de') + self.assertEqual(url_or_none('rtspu://foo.de'), 'rtspu://foo.de') + self.assertEqual(url_or_none('ftps://foo.de'), 'ftps://foo.de') def test_parse_age_limit(self): self.assertEqual(parse_age_limit(None), None) -- cgit v1.2.3 From 00dd0cd573c6ef8ca38dc73b88160b6c9f074dbe Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 8 Jan 2021 21:44:50 +0530 Subject: Update to ytdl-2021.01.08 --- test/test_utils.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index bb69b0522..a0f78ebe1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -21,6 +21,7 @@ from youtube_dlc.utils import ( encode_base_n, caesar, clean_html, + clean_podcast_url, date_from_str, DateRange, detect_exe_version, @@ -1497,6 +1498,10 @@ Line 1 iri_to_uri('http://导航.中国/'), 'http://xn--fet810g.xn--fiqs8s/') + def test_clean_podcast_url(self): + self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3') + self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3') + if __name__ == '__main__': unittest.main() -- cgit v1.2.3 From 7a5c1cfe93924351387b44919b3c0b2f66c4b883 Mon Sep 17 00:00:00 2001 From: Pccode66 <49125134+Pccode66@users.noreply.github.com> Date: Wed, 24 Feb 2021 15:45:56 -0300 Subject: Completely change project name to yt-dlp (#85) * All modules and binary names are changed * All documentation references changed * yt-dlp no longer loads youtube-dlc config files * All URLs changed to point to organization account Co-authored-by: Pccode66 Co-authored-by: pukkandan --- test/test_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index a0f78ebe1..795d2b46a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -15,7 +15,7 @@ import io import json import xml.etree.ElementTree -from youtube_dlc.utils import ( +from yt_dlp.utils import ( age_restricted, args_to_str, encode_base_n, @@ -107,7 +107,7 @@ from youtube_dlc.utils import ( parse_codecs, iri_to_uri, ) -from youtube_dlc.compat import ( +from yt_dlp.compat import ( compat_chr, compat_etree_fromstring, compat_getenv, @@ -242,12 +242,12 @@ class TestUtil(unittest.TestCase): def env(var): return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var) - compat_setenv('youtube_dlc_EXPATH_PATH', 'expanded') - self.assertEqual(expand_path(env('youtube_dlc_EXPATH_PATH')), 'expanded') + compat_setenv('yt_dlp_EXPATH_PATH', 'expanded') + self.assertEqual(expand_path(env('yt_dlp_EXPATH_PATH')), 'expanded') self.assertEqual(expand_path(env('HOME')), compat_getenv('HOME')) self.assertEqual(expand_path('~'), compat_getenv('HOME')) self.assertEqual( - expand_path('~/%s' % env('youtube_dlc_EXPATH_PATH')), + expand_path('~/%s' % env('yt_dlp_EXPATH_PATH')), '%s/expanded' % compat_getenv('HOME')) def test_prepend_extension(self): @@ -1425,8 +1425,8 @@ Line 1 self.assertEqual(caesar('ebg', 'acegik', -2), 'abc') def test_rot47(self): - self.assertEqual(rot47('youtube-dlc'), r'J@FEF36\5=4') - self.assertEqual(rot47('YOUTUBE-DLC'), r'*~&%&qt\s{r') + self.assertEqual(rot47('yt-dlp'), r'JE\5=A') + self.assertEqual(rot47('YT-DLP'), r'*%\s{!') def test_urshift(self): self.assertEqual(urshift(3, 1), 1) -- cgit v1.2.3 From 9e62f283ffd07ddb19de8b9f03db377aad369cc1 Mon Sep 17 00:00:00 2001 From: colethedj Date: Tue, 6 Apr 2021 18:45:15 +1200 Subject: [utils] Add `datetime_from_str` to parse relative time (#221) and `datetime_add_months` to accurately add/subtract months Authored by: colethedj --- test/test_utils.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 795d2b46a..d0571c6f2 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -23,6 +23,7 @@ from yt_dlp.utils import ( clean_html, clean_podcast_url, date_from_str, + datetime_from_str, DateRange, detect_exe_version, determine_ext, @@ -311,8 +312,18 @@ class TestUtil(unittest.TestCase): self.assertEqual(date_from_str('yesterday'), date_from_str('now-1day')) self.assertEqual(date_from_str('now+7day'), date_from_str('now+1week')) self.assertEqual(date_from_str('now+14day'), date_from_str('now+2week')) - self.assertEqual(date_from_str('now+365day'), date_from_str('now+1year')) - self.assertEqual(date_from_str('now+30day'), date_from_str('now+1month')) + self.assertEqual(date_from_str('20200229+365day'), date_from_str('20200229+1year')) + self.assertEqual(date_from_str('20210131+28day'), date_from_str('20210131+1month')) + + def test_datetime_from_str(self): + self.assertEqual(datetime_from_str('yesterday', precision='day'), datetime_from_str('now-1day', precision='auto')) + self.assertEqual(datetime_from_str('now+7day', precision='day'), datetime_from_str('now+1week', precision='auto')) + self.assertEqual(datetime_from_str('now+14day', precision='day'), datetime_from_str('now+2week', precision='auto')) + self.assertEqual(datetime_from_str('20200229+365day', precision='day'), datetime_from_str('20200229+1year', precision='auto')) + self.assertEqual(datetime_from_str('20210131+28day', precision='day'), datetime_from_str('20210131+1month', precision='auto')) + self.assertEqual(datetime_from_str('20210131+59day', precision='day'), datetime_from_str('20210131+2month', precision='auto')) + self.assertEqual(datetime_from_str('now+1day', precision='hour'), datetime_from_str('now+24hours', precision='auto')) + self.assertEqual(datetime_from_str('now+23hours', precision='hour'), datetime_from_str('now+23hours', precision='auto')) def test_daterange(self): _20century = DateRange("19000101", "20000101") -- cgit v1.2.3 From 5435dcf96ec444c92a402d0eb169d94015c0e6ba Mon Sep 17 00:00:00 2001 From: Hubert Hirtz Date: Mon, 19 Apr 2021 14:07:45 +0200 Subject: Handle Basic Auth `user:pass` in URLs Fixes https://github.com/ytdl-org/youtube-dl/issues/20258, https://github.com/ytdl-org/youtube-dl/issues/26211 Authored by: hhirtz, pukkandan --- test/test_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index d0571c6f2..a8666caab 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -66,6 +66,7 @@ from yt_dlp.utils import ( sanitize_filename, sanitize_path, sanitize_url, + sanitized_Request, expand_path, prepend_extension, replace_extension, @@ -239,6 +240,15 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_url('rmtps://foo.bar'), 'rtmps://foo.bar') self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar') + def test_extract_basic_auth(self): + auth_header = lambda url: sanitized_Request(url).get_header('Authorization') + self.assertFalse(auth_header('http://foo.bar')) + self.assertFalse(auth_header('http://:foo.bar')) + self.assertEqual(auth_header('http://@foo.bar'), 'Basic Og==') + self.assertEqual(auth_header('http://:pass@foo.bar'), 'Basic OnBhc3M=') + self.assertEqual(auth_header('http://user:@foo.bar'), 'Basic dXNlcjo=') + self.assertEqual(auth_header('http://user:pass@foo.bar'), 'Basic dXNlcjpwYXNz') + def test_expand_path(self): def env(var): return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var) -- cgit v1.2.3 From bc6b9bcd6554c10aa321cbfe151272e0df1a869b Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 1 Jun 2021 18:05:41 +0530 Subject: [utils] Escape URLs in `sanitized_Request`, not `sanitize_url` d2558234cf5dd12d6896eed5427b7dcdb3ab7b5a added escaping of URLs while sanitizing. However, `sanitize_url` may not always receive an actual URL. Eg: When using `yt-dlp "search query" --default-search ytsearch`, `search query` gets escaped to `search%20query` before being prefixed with `ytsearch:` which is not the intended behavior. So the escaping is moved to `sanitized_Request` instead. --- test/test_utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index a8666caab..cf541de4a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -239,6 +239,7 @@ class TestUtil(unittest.TestCase): self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar') self.assertEqual(sanitize_url('rmtps://foo.bar'), 'rtmps://foo.bar') self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar') + self.assertEqual(sanitize_url('foo bar'), 'foo bar') def test_extract_basic_auth(self): auth_header = lambda url: sanitized_Request(url).get_header('Authorization') -- cgit v1.2.3 From cc52de43568d8cd58c7e2ef4e5cecf609da28a9c Mon Sep 17 00:00:00 2001 From: felix Date: Thu, 3 Jun 2021 11:43:42 +0200 Subject: [cleanup] Point all shebang to `python3` (#372) Authored by: fstirlitz --- test/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index cf541de4a..04d355b4f 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # coding: utf-8 from __future__ import unicode_literals -- cgit v1.2.3 From 639f1cea9285d79c0eef4d2ec332b505c37ef34e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 9 Jun 2021 14:43:51 +0530 Subject: Fix `%d` and empty default in outtmpl Closes #388 --- test/test_utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 04d355b4f..9ff13a369 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -126,6 +126,7 @@ class TestUtil(unittest.TestCase): self.assertTrue(timeconvert('bougrg') is None) def test_sanitize_filename(self): + self.assertEqual(sanitize_filename(''), '') self.assertEqual(sanitize_filename('abc'), 'abc') self.assertEqual(sanitize_filename('abc_d-e'), 'abc_d-e') -- cgit v1.2.3 From 28419ca2c84de90acbfdb769d1a38440d93bd9c5 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 12 Jun 2021 20:44:30 +0530 Subject: [utils] Improve `LazyList` * Add `repr` and `str` that mimics `list` * Add `reversed`. Unlike `[::-1]`, reversed does not exhaust the iterable and modifies the `LazyList` in-place * Add tests --- test/test_utils.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 9ff13a369..ade10a7b1 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -12,6 +12,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Various small unit tests import io +import itertools import json import xml.etree.ElementTree @@ -108,6 +109,7 @@ from yt_dlp.utils import ( cli_bool_option, parse_codecs, iri_to_uri, + LazyList, ) from yt_dlp.compat import ( compat_chr, @@ -1525,6 +1527,47 @@ Line 1 self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3') self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3') + def test_LazyList(self): + it = list(range(10)) + + self.assertEqual(list(LazyList(it)), it) + self.assertEqual(LazyList(it).exhaust(), it) + self.assertEqual(LazyList(it)[5], it[5]) + + self.assertEqual(LazyList(it)[::2], it[::2]) + self.assertEqual(LazyList(it)[1::2], it[1::2]) + self.assertEqual(LazyList(it)[6:2:-2], it[6:2:-2]) + self.assertEqual(LazyList(it)[::-1], it[::-1]) + + self.assertTrue(LazyList(it)) + self.assertFalse(LazyList(range(0))) + self.assertEqual(len(LazyList(it)), len(it)) + self.assertEqual(repr(LazyList(it)), repr(it)) + self.assertEqual(str(LazyList(it)), str(it)) + + self.assertEqual(list(reversed(LazyList(it))), it[::-1]) + self.assertEqual(list(reversed(LazyList(it))[1:3:7]), it[::-1][1:3:7]) + + def test_LazyList_laziness(self): + + def test(ll, idx, val, cache): + self.assertEqual(ll[idx], val) + self.assertEqual(getattr(ll, '_LazyList__cache'), list(cache)) + + ll = LazyList(range(10)) + test(ll, 0, 0, range(1)) + test(ll, 5, 5, range(6)) + test(ll, -3, 7, range(10)) + + ll = reversed(LazyList(range(10))) + test(ll, -1, 0, range(1)) + test(ll, 3, 6, range(10)) + + ll = LazyList(itertools.count()) + test(ll, 10, 10, range(11)) + reversed(ll) + test(ll, -15, 14, range(15)) + if __name__ == '__main__': unittest.main() -- cgit v1.2.3 From 981052c9c6febb33b6547140a67a49ac0f5f4578 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 27 Jun 2021 07:35:58 +0530 Subject: Some minor fixes and refactoring (see desc) * [utils] Fix issues with reversal * check_formats should catch `DownloadError`, not `ExtractorError` * Simplify format selectors with `LazyList` and `yield from` --- test/test_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index ade10a7b1..0067e1ec9 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1545,8 +1545,8 @@ Line 1 self.assertEqual(repr(LazyList(it)), repr(it)) self.assertEqual(str(LazyList(it)), str(it)) - self.assertEqual(list(reversed(LazyList(it))), it[::-1]) - self.assertEqual(list(reversed(LazyList(it))[1:3:7]), it[::-1][1:3:7]) + self.assertEqual(list(LazyList(it).reverse()), it[::-1]) + self.assertEqual(list(LazyList(it).reverse()[1:3:7]), it[::-1][1:3:7]) def test_LazyList_laziness(self): @@ -1559,13 +1559,13 @@ Line 1 test(ll, 5, 5, range(6)) test(ll, -3, 7, range(10)) - ll = reversed(LazyList(range(10))) + ll = LazyList(range(10)).reverse() test(ll, -1, 0, range(1)) test(ll, 3, 6, range(10)) ll = LazyList(itertools.count()) test(ll, 10, 10, range(11)) - reversed(ll) + ll.reverse() test(ll, -15, 14, range(15)) -- cgit v1.2.3 From c843e685884ccc1a5186693d6450a34232b4377d Mon Sep 17 00:00:00 2001 From: felix Date: Tue, 13 Jul 2021 09:18:20 +0200 Subject: [utils] Improve `js_to_json` comment regex Capture the newline character as part of a single-line comment From #497, Authored by: fstirlitz --- test/test_utils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 0067e1ec9..4bfe250ac 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1054,6 +1054,9 @@ class TestUtil(unittest.TestCase): on = js_to_json('{ "040": "040" }') self.assertEqual(json.loads(on), {'040': '040'}) + on = js_to_json('[1,//{},\n2]') + self.assertEqual(json.loads(on), [1, 2]) + def test_js_to_json_malformed(self): self.assertEqual(js_to_json('42a1'), '42"a1"') self.assertEqual(js_to_json('42a-1'), '42"a"-1') -- cgit v1.2.3 From e0f2b4b47de79b0b7a1bf957dd18ed5b79eb4d49 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 1 Aug 2021 11:47:30 +0530 Subject: [utils] Fix slicing of reversed `LazyList` Closes #589 --- test/test_utils.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 4bfe250ac..f73e7b204 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1537,8 +1537,11 @@ Line 1 self.assertEqual(LazyList(it).exhaust(), it) self.assertEqual(LazyList(it)[5], it[5]) + self.assertEqual(LazyList(it)[5:], it[5:]) + self.assertEqual(LazyList(it)[:5], it[:5]) self.assertEqual(LazyList(it)[::2], it[::2]) self.assertEqual(LazyList(it)[1::2], it[1::2]) + self.assertEqual(LazyList(it)[5::-1], it[5::-1]) self.assertEqual(LazyList(it)[6:2:-2], it[6:2:-2]) self.assertEqual(LazyList(it)[::-1], it[::-1]) @@ -1550,6 +1553,7 @@ Line 1 self.assertEqual(list(LazyList(it).reverse()), it[::-1]) self.assertEqual(list(LazyList(it).reverse()[1:3:7]), it[::-1][1:3:7]) + self.assertEqual(list(LazyList(it).reverse()[::-1]), it) def test_LazyList_laziness(self): -- cgit v1.2.3 From 77b87f0519719c4264f400f5627da86c12f48bca Mon Sep 17 00:00:00 2001 From: Max Teegen <870074+max-te@users.noreply.github.com> Date: Sun, 13 Jun 2021 16:25:19 +0200 Subject: Add all format filtering operators also to `--match-filter` PR: https://github.com/ytdl-org/youtube-dl/pull/27361 Authored by: max-te --- test/test_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index f73e7b204..5ac5dedc9 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1207,7 +1207,6 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') '9999 51') def test_match_str(self): - self.assertRaises(ValueError, match_str, 'xy>foobar', {}) self.assertFalse(match_str('xy', {'x': 1200})) self.assertTrue(match_str('!xy', {'x': 1200})) self.assertTrue(match_str('x', {'x': 1200})) @@ -1224,6 +1223,17 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'})) self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'})) self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'})) + self.assertTrue(match_str('y^=foo', {'y': 'foobar42'})) + self.assertFalse(match_str('y!^=foo', {'y': 'foobar42'})) + self.assertFalse(match_str('y^=bar', {'y': 'foobar42'})) + self.assertTrue(match_str('y!^=bar', {'y': 'foobar42'})) + self.assertRaises(ValueError, match_str, 'x^=42', {'x': 42}) + self.assertTrue(match_str('y*=bar', {'y': 'foobar42'})) + self.assertFalse(match_str('y!*=bar', {'y': 'foobar42'})) + self.assertFalse(match_str('y*=baz', {'y': 'foobar42'})) + self.assertTrue(match_str('y!*=baz', {'y': 'foobar42'})) + self.assertTrue(match_str('y$=42', {'y': 'foobar42'})) + self.assertFalse(match_str('y$=43', {'y': 'foobar42'})) self.assertFalse(match_str( 'like_count > 100 & dislike_count Date: Thu, 5 Aug 2021 03:01:23 +0530 Subject: Add regex to `--match-filter` This does not fully deprecate `--match-title`/`--reject-title` since `--match-filter` is only checked after the extraction is complete, while `--match-title` can often be checked from the flat playlist. Fixes: https://github.com/ytdl-org/youtube-dl/issues/9092, https://github.com/ytdl-org/youtube-dl/issues/23035 --- test/test_utils.py | 54 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 12 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 5ac5dedc9..aef59e491 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1207,11 +1207,26 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') '9999 51') def test_match_str(self): + # Unary self.assertFalse(match_str('xy', {'x': 1200})) self.assertTrue(match_str('!xy', {'x': 1200})) self.assertTrue(match_str('x', {'x': 1200})) self.assertFalse(match_str('!x', {'x': 1200})) self.assertTrue(match_str('x', {'x': 0})) + self.assertTrue(match_str('is_live', {'is_live': True})) + self.assertFalse(match_str('is_live', {'is_live': False})) + self.assertFalse(match_str('is_live', {'is_live': None})) + self.assertFalse(match_str('is_live', {})) + self.assertFalse(match_str('!is_live', {'is_live': True})) + self.assertTrue(match_str('!is_live', {'is_live': False})) + self.assertTrue(match_str('!is_live', {'is_live': None})) + self.assertTrue(match_str('!is_live', {})) + self.assertTrue(match_str('title', {'title': 'abc'})) + self.assertTrue(match_str('title', {'title': ''})) + self.assertFalse(match_str('!title', {'title': 'abc'})) + self.assertFalse(match_str('!title', {'title': ''})) + + # Numeric self.assertFalse(match_str('x>0', {'x': 0})) self.assertFalse(match_str('x>0', {})) self.assertTrue(match_str('x>?0', {})) @@ -1219,6 +1234,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') self.assertFalse(match_str('x>2K', {'x': 1200})) self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200})) self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200})) + + # String self.assertFalse(match_str('y=a212', {'y': 'foobar42'})) self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'})) self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'})) @@ -1234,6 +1251,8 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') self.assertTrue(match_str('y!*=baz', {'y': 'foobar42'})) self.assertTrue(match_str('y$=42', {'y': 'foobar42'})) self.assertFalse(match_str('y$=43', {'y': 'foobar42'})) + + # And self.assertFalse(match_str( 'like_count > 100 & dislike_count 100 & dislike_count ?100 & description~=\'(?i)\bcats \& dogs\b\'', + {'description': 'Raining Cats & Dogs'}) def test_parse_dfxp_time_expr(self): self.assertEqual(parse_dfxp_time_expr(None), None) -- cgit v1.2.3 From 8f18aca8717bb0dd49054555af8d386e5eda3a88 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 15 Aug 2021 13:42:23 +0530 Subject: Let `--match-filter` reject entries early Makes redundant: `--match-title`, `--reject-title`, `--min-views`, `--max-views` --- test/test_utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index aef59e491..dedc598f7 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1285,9 +1285,15 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') self.assertTrue(match_str(r'x="foo \& bar" & x^=foo', {'x': 'foo & bar'})) # Example from docs - self.assertTrue( - r'!is_live & like_count>?100 & description~=\'(?i)\bcats \& dogs\b\'', - {'description': 'Raining Cats & Dogs'}) + self.assertTrue(match_str( + r"!is_live & like_count>?100 & description~='(?i)\bcats \& dogs\b'", + {'description': 'Raining Cats & Dogs'})) + + # Incomplete + self.assertFalse(match_str('id!=foo', {'id': 'foo'}, True)) + self.assertTrue(match_str('x', {'id': 'foo'}, True)) + self.assertTrue(match_str('!x', {'id': 'foo'}, True)) + self.assertFalse(match_str('x', {'id': 'foo'}, False)) def test_parse_dfxp_time_expr(self): self.assertEqual(parse_dfxp_time_expr(None), None) -- cgit v1.2.3 From 4dfbf8696b0c614eee480b44a86959b9a055fb97 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 23 Aug 2021 00:32:00 +0530 Subject: [utils] Add `parse_qs` --- test/test_utils.py | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index dedc598f7..d20bca795 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -62,6 +62,7 @@ from yt_dlp.utils import ( parse_iso8601, parse_resolution, parse_bitrate, + parse_qs, pkcs1pad, read_batch_urls, sanitize_filename, @@ -117,8 +118,6 @@ from yt_dlp.compat import ( compat_getenv, compat_os_name, compat_setenv, - compat_urlparse, - compat_parse_qs, ) @@ -688,38 +687,36 @@ class TestUtil(unittest.TestCase): self.assertTrue(isinstance(data, bytes)) def test_update_url_query(self): - def query_dict(url): - return compat_parse_qs(compat_urlparse.urlparse(url).query) - self.assertEqual(query_dict(update_url_query( + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})), - query_dict('http://example.com/path?quality=HD&format=mp4')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?quality=HD&format=mp4')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})), - query_dict('http://example.com/path?system=LINUX&system=WINDOWS')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?system=LINUX&system=WINDOWS')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'fields': 'id,formats,subtitles'})), - query_dict('http://example.com/path?fields=id,formats,subtitles')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?fields=id,formats,subtitles')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})), - query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path?manifest=f4m', {'manifest': []})), - query_dict('http://example.com/path')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})), - query_dict('http://example.com/path?system=LINUX')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?system=LINUX')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'fields': b'id,formats,subtitles'})), - query_dict('http://example.com/path?fields=id,formats,subtitles')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?fields=id,formats,subtitles')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'width': 1080, 'height': 720})), - query_dict('http://example.com/path?width=1080&height=720')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?width=1080&height=720')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'bitrate': 5020.43})), - query_dict('http://example.com/path?bitrate=5020.43')) - self.assertEqual(query_dict(update_url_query( + parse_qs('http://example.com/path?bitrate=5020.43')) + self.assertEqual(parse_qs(update_url_query( 'http://example.com/path', {'test': '第二行тест'})), - query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82')) + parse_qs('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82')) def test_multipart_encode(self): self.assertEqual( -- cgit v1.2.3 From 18f96d129b24200debf257153bcc762125d2a1f7 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 17 Oct 2021 01:04:00 +0530 Subject: [utils] Allow duration strings in filter Closes #1309 --- test/test_utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index d20bca795..7fc431505 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1231,6 +1231,7 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4') self.assertFalse(match_str('x>2K', {'x': 1200})) self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200})) self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200})) + self.assertTrue(match_str('x > 1:0:0', {'x': 3700})) # String self.assertFalse(match_str('y=a212', {'y': 'foobar42'})) -- cgit v1.2.3 From 176f1866cb437dd59cf8f600638cfd7ba2a8525e Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 18 Oct 2021 18:34:21 +0530 Subject: Add HDR information to formats --- test/test_utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'test/test_utils.py') diff --git a/test/test_utils.py b/test/test_utils.py index 7fc431505..9a5e3f0f0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -848,30 +848,52 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_codecs('avc1.77.30, mp4a.40.2'), { 'vcodec': 'avc1.77.30', 'acodec': 'mp4a.40.2', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('mp4a.40.2'), { 'vcodec': 'none', 'acodec': 'mp4a.40.2', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('mp4a.40.5,avc1.42001e'), { 'vcodec': 'avc1.42001e', 'acodec': 'mp4a.40.5', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('avc3.640028'), { 'vcodec': 'avc3.640028', 'acodec': 'none', + 'dynamic_range': None, }) self.assertEqual(parse_codecs(', h264,,newcodec,aac'), { 'vcodec': 'h264', 'acodec': 'aac', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('av01.0.05M.08'), { 'vcodec': 'av01.0.05M.08', 'acodec': 'none', + 'dynamic_range': None, + }) + self.assertEqual(parse_codecs('vp9.2'), { + 'vcodec': 'vp9.2', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('av01.0.12M.10.0.110.09.16.09.0'), { + 'vcodec': 'av01.0.12M.10', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('dvhe'), { + 'vcodec': 'dvhe', + 'acodec': 'none', + 'dynamic_range': 'DV', }) self.assertEqual(parse_codecs('theora, vorbis'), { 'vcodec': 'theora', 'acodec': 'vorbis', + 'dynamic_range': None, }) self.assertEqual(parse_codecs('unknownvcodec, unknownacodec'), { 'vcodec': 'unknownvcodec', -- cgit v1.2.3