aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorcoletdjnz <coletdjnz@protonmail.com>2023-07-09 13:23:02 +0530
committerpukkandan <pukkandan.ytdlp@gmail.com>2023-07-15 16:18:35 +0530
commit3d2623a898196640f7cc0fc8b70118ff19e6925d (patch)
treea0dc9fe53959ca673294902f7a553f55706cc5f3
parent227bf1a33be7b89cd7d44ad046844c4ccba104f4 (diff)
downloadhypervideo-pre-3d2623a898196640f7cc0fc8b70118ff19e6925d.tar.lz
hypervideo-pre-3d2623a898196640f7cc0fc8b70118ff19e6925d.tar.xz
hypervideo-pre-3d2623a898196640f7cc0fc8b70118ff19e6925d.zip
[compat, networking] Deprecate old functions (#2861)
Authored by: coletdjnz, pukkandan
-rwxr-xr-xtest/test_download.py2
-rw-r--r--test/test_networking.py27
-rw-r--r--test/test_networking_utils.py64
-rw-r--r--yt_dlp/YoutubeDL.py12
-rw-r--r--yt_dlp/__init__.py2
-rw-r--r--yt_dlp/compat/_deprecated.py1
-rw-r--r--yt_dlp/compat/_legacy.py1
-rw-r--r--yt_dlp/downloader/external.py7
-rw-r--r--yt_dlp/downloader/f4m.py8
-rw-r--r--yt_dlp/downloader/fragment.py19
-rw-r--r--yt_dlp/downloader/hls.py2
-rw-r--r--yt_dlp/downloader/http.py41
-rw-r--r--yt_dlp/downloader/ism.py4
-rw-r--r--yt_dlp/downloader/niconico.py11
-rw-r--r--yt_dlp/downloader/youtube_live_chat.py10
-rw-r--r--yt_dlp/extractor/abematv.py3
-rw-r--r--yt_dlp/extractor/adn.py16
-rw-r--r--yt_dlp/extractor/adobepass.py20
-rw-r--r--yt_dlp/extractor/ant1newsgr.py4
-rw-r--r--yt_dlp/extractor/archiveorg.py12
-rw-r--r--yt_dlp/extractor/atresplayer.py6
-rw-r--r--yt_dlp/extractor/bbc.py14
-rw-r--r--yt_dlp/extractor/bilibili.py4
-rw-r--r--yt_dlp/extractor/bitchute.py2
-rw-r--r--yt_dlp/extractor/bravotv.py4
-rw-r--r--yt_dlp/extractor/brightcove.py6
-rw-r--r--yt_dlp/extractor/canalplus.py2
-rw-r--r--yt_dlp/extractor/cbsnews.py2
-rw-r--r--yt_dlp/extractor/ceskatelevize.py30
-rw-r--r--yt_dlp/extractor/cinetecamilano.py4
-rw-r--r--yt_dlp/extractor/ciscowebex.py6
-rw-r--r--yt_dlp/extractor/common.py41
-rw-r--r--yt_dlp/extractor/crackle.py4
-rw-r--r--yt_dlp/extractor/crunchyroll.py4
-rw-r--r--yt_dlp/extractor/cultureunplugged.py6
-rw-r--r--yt_dlp/extractor/dacast.py4
-rw-r--r--yt_dlp/extractor/dailymotion.py6
-rw-r--r--yt_dlp/extractor/discovery.py6
-rw-r--r--yt_dlp/extractor/dplay.py8
-rw-r--r--yt_dlp/extractor/eagleplatform.py6
-rw-r--r--yt_dlp/extractor/eitb.py10
-rw-r--r--yt_dlp/extractor/eporner.py2
-rw-r--r--yt_dlp/extractor/facebook.py14
-rw-r--r--yt_dlp/extractor/fc2.py6
-rw-r--r--yt_dlp/extractor/filmon.py14
-rw-r--r--yt_dlp/extractor/fox.py10
-rw-r--r--yt_dlp/extractor/foxsports.py5
-rw-r--r--yt_dlp/extractor/fujitv.py2
-rw-r--r--yt_dlp/extractor/funimation.py6
-rw-r--r--yt_dlp/extractor/gdcvault.py15
-rw-r--r--yt_dlp/extractor/generic.py8
-rw-r--r--yt_dlp/extractor/globo.py2
-rw-r--r--yt_dlp/extractor/googledrive.py2
-rw-r--r--yt_dlp/extractor/hketv.py2
-rw-r--r--yt_dlp/extractor/hotnewhiphop.py14
-rw-r--r--yt_dlp/extractor/hotstar.py5
-rw-r--r--yt_dlp/extractor/hrti.py10
-rw-r--r--yt_dlp/extractor/ign.py17
-rw-r--r--yt_dlp/extractor/imggaming.py6
-rw-r--r--yt_dlp/extractor/instagram.py6
-rw-r--r--yt_dlp/extractor/iprima.py4
-rw-r--r--yt_dlp/extractor/kakao.py6
-rw-r--r--yt_dlp/extractor/kick.py3
-rw-r--r--yt_dlp/extractor/kuwo.py2
-rw-r--r--yt_dlp/extractor/la7.py9
-rw-r--r--yt_dlp/extractor/lbry.py4
-rw-r--r--yt_dlp/extractor/lecturio.py2
-rw-r--r--yt_dlp/extractor/lego.py4
-rw-r--r--yt_dlp/extractor/limelight.py6
-rw-r--r--yt_dlp/extractor/linuxacademy.py15
-rw-r--r--yt_dlp/extractor/mediasite.py2
-rw-r--r--yt_dlp/extractor/megatvcom.py6
-rw-r--r--yt_dlp/extractor/mgtv.py6
-rw-r--r--yt_dlp/extractor/minds.py2
-rw-r--r--yt_dlp/extractor/miomio.py10
-rw-r--r--yt_dlp/extractor/mtv.py11
-rw-r--r--yt_dlp/extractor/nbc.py2
-rw-r--r--yt_dlp/extractor/nebula.py4
-rw-r--r--yt_dlp/extractor/neteasemusic.py6
-rw-r--r--yt_dlp/extractor/niconico.py16
-rw-r--r--yt_dlp/extractor/njpwworld.py2
-rw-r--r--yt_dlp/extractor/nosvideo.py6
-rw-r--r--yt_dlp/extractor/nowness.py8
-rw-r--r--yt_dlp/extractor/nrk.py5
-rw-r--r--yt_dlp/extractor/odkmedia.py6
-rw-r--r--yt_dlp/extractor/odnoklassniki.py4
-rw-r--r--yt_dlp/extractor/orf.py2
-rw-r--r--yt_dlp/extractor/owncloud.py2
-rw-r--r--yt_dlp/extractor/packtpub.py11
-rw-r--r--yt_dlp/extractor/patreon.py6
-rw-r--r--yt_dlp/extractor/peloton.py12
-rw-r--r--yt_dlp/extractor/piapro.py2
-rw-r--r--yt_dlp/extractor/pladform.py2
-rw-r--r--yt_dlp/extractor/platzi.py2
-rw-r--r--yt_dlp/extractor/playplustv.py14
-rw-r--r--yt_dlp/extractor/pornhub.py11
-rw-r--r--yt_dlp/extractor/puhutv.py8
-rw-r--r--yt_dlp/extractor/radiko.py2
-rw-r--r--yt_dlp/extractor/radiocanada.py6
-rw-r--r--yt_dlp/extractor/rcs.py2
-rw-r--r--yt_dlp/extractor/rcti.py4
-rw-r--r--yt_dlp/extractor/recurbate.py5
-rw-r--r--yt_dlp/extractor/redbulltv.py6
-rw-r--r--yt_dlp/extractor/redgifs.py4
-rw-r--r--yt_dlp/extractor/regiotv.py10
-rw-r--r--yt_dlp/extractor/rokfin.py4
-rw-r--r--yt_dlp/extractor/roosterteeth.py10
-rw-r--r--yt_dlp/extractor/rozhlas.py4
-rw-r--r--yt_dlp/extractor/rte.py6
-rw-r--r--yt_dlp/extractor/rts.py4
-rw-r--r--yt_dlp/extractor/rumble.py4
-rw-r--r--yt_dlp/extractor/safari.py6
-rw-r--r--yt_dlp/extractor/sbs.py2
-rw-r--r--yt_dlp/extractor/sevenplus.py10
-rw-r--r--yt_dlp/extractor/shahid.py8
-rw-r--r--yt_dlp/extractor/sina.py10
-rw-r--r--yt_dlp/extractor/sixplay.py2
-rw-r--r--yt_dlp/extractor/slideslive.py2
-rw-r--r--yt_dlp/extractor/sonyliv.py10
-rw-r--r--yt_dlp/extractor/soundcloud.py17
-rw-r--r--yt_dlp/extractor/teachable.py2
-rw-r--r--yt_dlp/extractor/telemundo.py9
-rw-r--r--yt_dlp/extractor/tennistv.py2
-rw-r--r--yt_dlp/extractor/tenplay.py9
-rw-r--r--yt_dlp/extractor/tfo.py8
-rw-r--r--yt_dlp/extractor/theplatform.py4
-rw-r--r--yt_dlp/extractor/thisoldhouse.py4
-rw-r--r--yt_dlp/extractor/threeqsdn.py4
-rw-r--r--yt_dlp/extractor/tiktok.py4
-rw-r--r--yt_dlp/extractor/toutv.py6
-rw-r--r--yt_dlp/extractor/triller.py4
-rw-r--r--yt_dlp/extractor/trueid.py6
-rw-r--r--yt_dlp/extractor/tubetugraz.py8
-rw-r--r--yt_dlp/extractor/tubitv.py8
-rw-r--r--yt_dlp/extractor/tumblr.py2
-rw-r--r--yt_dlp/extractor/tunein.py4
-rw-r--r--yt_dlp/extractor/tv2.py10
-rw-r--r--yt_dlp/extractor/tvp.py4
-rw-r--r--yt_dlp/extractor/tvplay.py10
-rw-r--r--yt_dlp/extractor/tvplayer.py10
-rw-r--r--yt_dlp/extractor/twitcasting.py4
-rw-r--r--yt_dlp/extractor/twitch.py2
-rw-r--r--yt_dlp/extractor/twitter.py2
-rw-r--r--yt_dlp/extractor/udemy.py15
-rw-r--r--yt_dlp/extractor/vevo.py10
-rw-r--r--yt_dlp/extractor/vice.py10
-rw-r--r--yt_dlp/extractor/videocampus_sachsen.py4
-rw-r--r--yt_dlp/extractor/vidio.py2
-rw-r--r--yt_dlp/extractor/vidlii.py2
-rw-r--r--yt_dlp/extractor/viewlift.py6
-rw-r--r--yt_dlp/extractor/viidea.py6
-rw-r--r--yt_dlp/extractor/vimeo.py31
-rw-r--r--yt_dlp/extractor/vk.py2
-rw-r--r--yt_dlp/extractor/vocaroo.py6
-rw-r--r--yt_dlp/extractor/vodlocker.py12
-rw-r--r--yt_dlp/extractor/voot.py4
-rw-r--r--yt_dlp/extractor/vrt.py4
-rw-r--r--yt_dlp/extractor/vrv.py7
-rw-r--r--yt_dlp/extractor/weibo.py2
-rw-r--r--yt_dlp/extractor/weverse.py8
-rw-r--r--yt_dlp/extractor/wistia.py6
-rw-r--r--yt_dlp/extractor/wykop.py4
-rw-r--r--yt_dlp/extractor/xhamster.py2
-rw-r--r--yt_dlp/extractor/xtube.py4
-rw-r--r--yt_dlp/extractor/yesjapan.py9
-rw-r--r--yt_dlp/extractor/youtube.py15
-rw-r--r--yt_dlp/extractor/zaiko.py2
-rw-r--r--yt_dlp/extractor/zattoo.py5
-rw-r--r--yt_dlp/extractor/zype.py6
-rw-r--r--yt_dlp/networking/common.py6
-rw-r--r--yt_dlp/networking/exceptions.py22
-rw-r--r--yt_dlp/postprocessor/common.py11
-rw-r--r--yt_dlp/update.py9
-rw-r--r--yt_dlp/utils/_deprecated.py19
-rw-r--r--yt_dlp/utils/_legacy.py62
-rw-r--r--yt_dlp/utils/_utils.py47
176 files changed, 706 insertions, 728 deletions
diff --git a/test/test_download.py b/test/test_download.py
index fd7752cdd..6f00a4ded 100755
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -160,7 +160,7 @@ def generator(test_case, tname):
force_generic_extractor=params.get('force_generic_extractor', False))
except (DownloadError, ExtractorError) as err:
# Check if the exception is not a network related one
- if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].code == 503):
+ if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].status == 503):
err.msg = f'{getattr(err, "msg", err)} ({tname})'
raise
diff --git a/test/test_networking.py b/test/test_networking.py
index 147a4ff49..b60ed283b 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -1057,14 +1057,15 @@ class TestYoutubeDLNetworking:
urllib_req = urllib.request.Request('http://foo.bar', data=b'test', method='PUT', headers={'X-Test': '1'})
urllib_req.add_unredirected_header('Cookie', 'bob=bob')
urllib_req.timeout = 2
-
- req = ydl.urlopen(urllib_req).request
- assert req.url == urllib_req.get_full_url()
- assert req.data == urllib_req.data
- assert req.method == urllib_req.get_method()
- assert 'X-Test' in req.headers
- assert 'Cookie' in req.headers
- assert req.extensions.get('timeout') == 2
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ req = ydl.urlopen(urllib_req).request
+ assert req.url == urllib_req.get_full_url()
+ assert req.data == urllib_req.data
+ assert req.method == urllib_req.get_method()
+ assert 'X-Test' in req.headers
+ assert 'Cookie' in req.headers
+ assert req.extensions.get('timeout') == 2
with pytest.raises(AssertionError):
ydl.urlopen(None)
@@ -1362,7 +1363,9 @@ class TestResponse:
def test_compat(self):
res = Response(io.BytesIO(b''), url='test://', status=404, headers={'test': 'test'})
- assert res.code == res.getcode() == res.status
- assert res.geturl() == res.url
- assert res.info() is res.headers
- assert res.getheader('test') == res.get_header('test')
+ with warnings.catch_warnings():
+ warnings.simplefilter('ignore', category=DeprecationWarning)
+ assert res.code == res.getcode() == res.status
+ assert res.geturl() == res.url
+ assert res.info() is res.headers
+ assert res.getheader('test') == res.get_header('test')
diff --git a/test/test_networking_utils.py b/test/test_networking_utils.py
index f9f876af3..ef46f79ed 100644
--- a/test/test_networking_utils.py
+++ b/test/test_networking_utils.py
@@ -8,11 +8,13 @@ import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import contextlib
import io
import platform
import random
import ssl
import urllib.error
+import warnings
from yt_dlp.cookies import YoutubeDLCookieJar
from yt_dlp.dependencies import certifi
@@ -202,20 +204,58 @@ class TestNetworkingExceptions:
assert isinstance(error, HTTPError)
assert isinstance(error, urllib.error.HTTPError)
- assert error.code == 403
- assert error.getcode() == 403
- assert error.hdrs is error.response.headers
- assert error.info() is error.response.headers
- assert error.headers is error.response.headers
- assert error.filename == error.response.url
- assert error.url == error.response.url
- assert error.geturl() == error.response.url
+ @contextlib.contextmanager
+ def raises_deprecation_warning():
+ with warnings.catch_warnings(record=True) as w:
+ warnings.simplefilter('always')
+ yield
+
+ if len(w) == 0:
+ pytest.fail('Did not raise DeprecationWarning')
+ if len(w) > 1:
+ pytest.fail(f'Raised multiple warnings: {w}')
+
+ if not issubclass(w[-1].category, DeprecationWarning):
+ pytest.fail(f'Expected DeprecationWarning, got {w[-1].category}')
+ w.clear()
+
+ with raises_deprecation_warning():
+ assert error.code == 403
+
+ with raises_deprecation_warning():
+ assert error.getcode() == 403
+
+ with raises_deprecation_warning():
+ assert error.hdrs is error.response.headers
+
+ with raises_deprecation_warning():
+ assert error.info() is error.response.headers
+
+ with raises_deprecation_warning():
+ assert error.headers is error.response.headers
+
+ with raises_deprecation_warning():
+ assert error.filename == error.response.url
+
+ with raises_deprecation_warning():
+ assert error.url == error.response.url
+
+ with raises_deprecation_warning():
+ assert error.geturl() == error.response.url
# Passthrough file operations
- assert error.read() == b'test'
- assert not error.closed
- # Technically Response operations are also passed through, which should not be used.
- assert error.get_header('test') == 'test'
+ with raises_deprecation_warning():
+ assert error.read() == b'test'
+
+ with raises_deprecation_warning():
+ assert not error.closed
+
+ with raises_deprecation_warning():
+ # Technically Response operations are also passed through, which should not be used.
+ assert error.get_header('test') == 'test'
+
+ # Should not raise a warning
+ error.close()
@pytest.mark.skipif(
platform.python_implementation() == 'PyPy', reason='garbage collector works differently in pypy')
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 29a18aef0..850eb8ae0 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -33,7 +33,7 @@ from .extractor import gen_extractor_classes, get_info_extractor
from .extractor.common import UnsupportedURLIE
from .extractor.openload import PhantomJSwrapper
from .minicurses import format_text
-from .networking import Request, RequestDirector
+from .networking import HEADRequest, Request, RequestDirector
from .networking.common import _REQUEST_HANDLERS
from .networking.exceptions import (
HTTPError,
@@ -41,6 +41,7 @@ from .networking.exceptions import (
RequestError,
SSLError,
_CompatHTTPError,
+ network_exceptions,
)
from .plugins import directories as plugin_directories
from .postprocessor import _PLUGIN_CLASSES as plugin_pps
@@ -80,7 +81,6 @@ from .utils import (
ExtractorError,
FormatSorter,
GeoRestrictedError,
- HEADRequest,
ISO3166Utils,
LazyList,
MaxDownloadsReached,
@@ -122,7 +122,6 @@ from .utils import (
locked_file,
make_archive_id,
make_dir,
- network_exceptions,
number_of_digits,
orderedSet,
orderedSet_from_options,
@@ -135,7 +134,6 @@ from .utils import (
sanitize_filename,
sanitize_path,
sanitize_url,
- std_headers,
str_or_none,
strftime_or_none,
subtitles_filename,
@@ -158,6 +156,7 @@ from .utils.networking import (
HTTPHeaderDict,
clean_headers,
clean_proxies,
+ std_headers,
)
from .version import CHANNEL, RELEASE_GIT_HEAD, VARIANT, __version__
@@ -4019,6 +4018,9 @@ class YoutubeDL:
if isinstance(req, str):
req = Request(req)
elif isinstance(req, urllib.request.Request):
+ self.deprecation_warning(
+ 'Passing a urllib.request.Request object to YoutubeDL.urlopen() is deprecated. '
+ 'Use yt_dlp.networking.common.Request instead.')
req = urllib_req_to_req(req)
assert isinstance(req, Request)
@@ -4242,7 +4244,7 @@ class YoutubeDL:
ret.append((thumb_filename, thumb_filename_final))
t['filepath'] = thumb_filename
except network_exceptions as err:
- if isinstance(err, urllib.error.HTTPError) and err.code == 404:
+ if isinstance(err, HTTPError) and err.status == 404:
self.to_screen(f'[info] {thumb_display_id.title()} does not exist')
else:
self.report_warning(f'Unable to download {thumb_display_id}: {err}')
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index b81277a57..991dbcda7 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -57,11 +57,11 @@ from .utils import (
read_stdin,
render_table,
setproctitle,
- std_headers,
traverse_obj,
variadic,
write_string,
)
+from .utils.networking import std_headers
from .YoutubeDL import YoutubeDL
_IN_CLI = False
diff --git a/yt_dlp/compat/_deprecated.py b/yt_dlp/compat/_deprecated.py
index 342f1f80d..14d37b236 100644
--- a/yt_dlp/compat/_deprecated.py
+++ b/yt_dlp/compat/_deprecated.py
@@ -8,7 +8,6 @@ compat_str = str
compat_b64decode = base64.b64decode
-compat_HTTPError = urllib.error.HTTPError
compat_urlparse = urllib.parse
compat_parse_qs = urllib.parse.parse_qs
compat_urllib_parse_unquote = urllib.parse.unquote
diff --git a/yt_dlp/compat/_legacy.py b/yt_dlp/compat/_legacy.py
index 83bf869a8..912907a02 100644
--- a/yt_dlp/compat/_legacy.py
+++ b/yt_dlp/compat/_legacy.py
@@ -70,6 +70,7 @@ compat_html_parser_HTMLParseError = compat_HTMLParseError
compat_HTMLParser = compat_html_parser_HTMLParser = html.parser.HTMLParser
compat_http_client = http.client
compat_http_server = http.server
+compat_HTTPError = urllib.error.HTTPError
compat_input = input
compat_integer_types = (int, )
compat_itertools_count = itertools.count
diff --git a/yt_dlp/downloader/external.py b/yt_dlp/downloader/external.py
index d4045e58f..e307502db 100644
--- a/yt_dlp/downloader/external.py
+++ b/yt_dlp/downloader/external.py
@@ -10,6 +10,7 @@ import uuid
from .fragment import FragmentFD
from ..compat import functools
+from ..networking import Request
from ..postprocessor.ffmpeg import EXT_TO_OUT_FORMATS, FFmpegPostProcessor
from ..utils import (
Popen,
@@ -25,7 +26,6 @@ from ..utils import (
encodeFilename,
find_available_port,
remove_end,
- sanitized_Request,
traverse_obj,
)
@@ -357,13 +357,12 @@ class Aria2cFD(ExternalFD):
'method': method,
'params': [f'token:{rpc_secret}', *params],
}).encode('utf-8')
- request = sanitized_Request(
+ request = Request(
f'http://localhost:{rpc_port}/jsonrpc',
data=d, headers={
'Content-Type': 'application/json',
'Content-Length': f'{len(d)}',
- 'Ytdl-request-proxy': '__noproxy__',
- })
+ }, proxies={'all': None})
with self.ydl.urlopen(request) as r:
resp = json.load(r)
assert resp.get('id') == sanitycheck, 'Something went wrong with RPC server'
diff --git a/yt_dlp/downloader/f4m.py b/yt_dlp/downloader/f4m.py
index 306f92192..28cbba016 100644
--- a/yt_dlp/downloader/f4m.py
+++ b/yt_dlp/downloader/f4m.py
@@ -3,11 +3,11 @@ import io
import itertools
import struct
import time
-import urllib.error
import urllib.parse
from .fragment import FragmentFD
from ..compat import compat_etree_fromstring
+from ..networking.exceptions import HTTPError
from ..utils import fix_xml_ampersands, xpath_text
@@ -312,7 +312,7 @@ class F4mFD(FragmentFD):
self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
- man_url = urlh.geturl()
+ man_url = urlh.url
# Some manifests may be malformed, e.g. prosiebensat1 generated manifests
# (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244
# and https://github.com/ytdl-org/youtube-dl/issues/7823)
@@ -407,8 +407,8 @@ class F4mFD(FragmentFD):
if box_type == b'mdat':
self._append_fragment(ctx, box_data)
break
- except urllib.error.HTTPError as err:
- if live and (err.code == 404 or err.code == 410):
+ except HTTPError as err:
+ if live and (err.status == 404 or err.status == 410):
# We didn't keep up with the live window. Continue
# with the next available fragment.
msg = 'Fragment %d unavailable' % frag_i
diff --git a/yt_dlp/downloader/fragment.py b/yt_dlp/downloader/fragment.py
index 069815326..b4b680dae 100644
--- a/yt_dlp/downloader/fragment.py
+++ b/yt_dlp/downloader/fragment.py
@@ -1,24 +1,19 @@
import concurrent.futures
import contextlib
-import http.client
import json
import math
import os
import struct
import time
-import urllib.error
from .common import FileDownloader
from .http import HttpFD
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
from ..compat import compat_os_name
-from ..utils import (
- DownloadError,
- RetryManager,
- encodeFilename,
- sanitized_Request,
- traverse_obj,
-)
+from ..networking import Request
+from ..networking.exceptions import HTTPError, IncompleteRead
+from ..utils import DownloadError, RetryManager, encodeFilename, traverse_obj
+from ..utils.networking import HTTPHeaderDict
class HttpQuietDownloader(HttpFD):
@@ -75,7 +70,7 @@ class FragmentFD(FileDownloader):
def _prepare_url(self, info_dict, url):
headers = info_dict.get('http_headers')
- return sanitized_Request(url, None, headers) if headers else url
+ return Request(url, None, headers) if headers else url
def _prepare_and_start_frag_download(self, ctx, info_dict):
self._prepare_frag_download(ctx)
@@ -457,7 +452,7 @@ class FragmentFD(FileDownloader):
frag_index = ctx['fragment_index'] = fragment['frag_index']
ctx['last_error'] = None
- headers = info_dict.get('http_headers', {}).copy()
+ headers = HTTPHeaderDict(info_dict.get('http_headers'))
byte_range = fragment.get('byte_range')
if byte_range:
headers['Range'] = 'bytes=%d-%d' % (byte_range['start'], byte_range['end'] - 1)
@@ -477,7 +472,7 @@ class FragmentFD(FileDownloader):
if not self._download_fragment(
ctx, fragment['url'], info_dict, headers, info_dict.get('request_data')):
return
- except (urllib.error.HTTPError, http.client.IncompleteRead) as err:
+ except (HTTPError, IncompleteRead) as err:
retry.error = err
continue
except DownloadError: # has own retry settings
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index ab7d496d4..d4b3f0320 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -75,7 +75,7 @@ class HlsFD(FragmentFD):
self.to_screen('[%s] Downloading m3u8 manifest' % self.FD_NAME)
urlh = self.ydl.urlopen(self._prepare_url(info_dict, man_url))
- man_url = urlh.geturl()
+ man_url = urlh.url
s = urlh.read().decode('utf-8', 'ignore')
can_download, message = self.can_download(s, info_dict, self.params.get('allow_unplayable_formats')), None
diff --git a/yt_dlp/downloader/http.py b/yt_dlp/downloader/http.py
index 45d094721..f5237443e 100644
--- a/yt_dlp/downloader/http.py
+++ b/yt_dlp/downloader/http.py
@@ -1,10 +1,14 @@
import os
import random
import time
-import urllib.error
from .common import FileDownloader
-from ..networking.exceptions import CertificateVerifyError, TransportError
+from ..networking import Request
+from ..networking.exceptions import (
+ CertificateVerifyError,
+ HTTPError,
+ TransportError,
+)
from ..utils import (
ContentTooShortError,
RetryManager,
@@ -14,10 +18,10 @@ from ..utils import (
encodeFilename,
int_or_none,
parse_http_range,
- sanitized_Request,
try_call,
write_xattr,
)
+from ..utils.networking import HTTPHeaderDict
class HttpFD(FileDownloader):
@@ -36,10 +40,7 @@ class HttpFD(FileDownloader):
ctx.stream = None
# Disable compression
- headers = {'Accept-Encoding': 'identity'}
- add_headers = info_dict.get('http_headers')
- if add_headers:
- headers.update(add_headers)
+ headers = HTTPHeaderDict({'Accept-Encoding': 'identity'}, info_dict.get('http_headers'))
is_test = self.params.get('test', False)
chunk_size = self._TEST_FILE_SIZE if is_test else (
@@ -110,10 +111,10 @@ class HttpFD(FileDownloader):
if try_call(lambda: range_end >= ctx.content_len):
range_end = ctx.content_len - 1
- request = sanitized_Request(url, request_data, headers)
+ request = Request(url, request_data, headers)
has_range = range_start is not None
if has_range:
- request.add_header('Range', f'bytes={int(range_start)}-{int_or_none(range_end) or ""}')
+ request.headers['Range'] = f'bytes={int(range_start)}-{int_or_none(range_end) or ""}'
# Establish connection
try:
ctx.data = self.ydl.urlopen(request)
@@ -144,17 +145,17 @@ class HttpFD(FileDownloader):
self.report_unable_to_resume()
ctx.resume_len = 0
ctx.open_mode = 'wb'
- ctx.data_len = ctx.content_len = int_or_none(ctx.data.info().get('Content-length', None))
- except urllib.error.HTTPError as err:
- if err.code == 416:
+ ctx.data_len = ctx.content_len = int_or_none(ctx.data.headers.get('Content-length', None))
+ except HTTPError as err:
+ if err.status == 416:
# Unable to resume (requested range not satisfiable)
try:
# Open the connection again without the range header
ctx.data = self.ydl.urlopen(
- sanitized_Request(url, request_data, headers))
- content_length = ctx.data.info()['Content-Length']
- except urllib.error.HTTPError as err:
- if err.code < 500 or err.code >= 600:
+ Request(url, request_data, headers))
+ content_length = ctx.data.headers['Content-Length']
+ except HTTPError as err:
+ if err.status < 500 or err.status >= 600:
raise
else:
# Examine the reported length
@@ -182,7 +183,7 @@ class HttpFD(FileDownloader):
ctx.resume_len = 0
ctx.open_mode = 'wb'
return
- elif err.code < 500 or err.code >= 600:
+ elif err.status < 500 or err.status >= 600:
# Unexpected HTTP error
raise
raise RetryDownload(err)
@@ -198,9 +199,9 @@ class HttpFD(FileDownloader):
ctx.stream = None
def download():
- data_len = ctx.data.info().get('Content-length')
+ data_len = ctx.data.headers.get('Content-length')
- if ctx.data.info().get('Content-encoding'):
+ if ctx.data.headers.get('Content-encoding'):
# Content-encoding is present, Content-length is not reliable anymore as we are
# doing auto decompression. (See: https://github.com/yt-dlp/yt-dlp/pull/6176)
data_len = None
@@ -345,7 +346,7 @@ class HttpFD(FileDownloader):
# Update file modification time
if self.params.get('updatetime', True):
- info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.info().get('last-modified', None))
+ info_dict['filetime'] = self.try_utime(ctx.filename, ctx.data.headers.get('last-modified', None))
self._hook_progress({
'downloaded_bytes': byte_counter,
diff --git a/yt_dlp/downloader/ism.py b/yt_dlp/downloader/ism.py
index a157a8ad9..dd688f586 100644
--- a/yt_dlp/downloader/ism.py
+++ b/yt_dlp/downloader/ism.py
@@ -2,9 +2,9 @@ import binascii
import io
import struct
import time
-import urllib.error
from .fragment import FragmentFD
+from ..networking.exceptions import HTTPError
from ..utils import RetryManager
u8 = struct.Struct('>B')
@@ -271,7 +271,7 @@ class IsmFD(FragmentFD):
write_piff_header(ctx['dest_stream'], info_dict['_download_params'])
extra_state['ism_track_written'] = True
self._append_fragment(ctx, frag_content)
- except urllib.error.HTTPError as err:
+ except HTTPError as err:
retry.error = err
continue
diff --git a/yt_dlp/downloader/niconico.py b/yt_dlp/downloader/niconico.py
index 7d8575c2a..5720f6eb8 100644
--- a/yt_dlp/downloader/niconico.py
+++ b/yt_dlp/downloader/niconico.py
@@ -5,13 +5,8 @@ import time
from . import get_suitable_downloader
from .common import FileDownloader
from .external import FFmpegFD
-from ..utils import (
- DownloadError,
- WebSocketsWrapper,
- sanitized_Request,
- str_or_none,
- try_get,
-)
+from ..networking import Request
+from ..utils import DownloadError, WebSocketsWrapper, str_or_none, try_get
class NiconicoDmcFD(FileDownloader):
@@ -33,7 +28,7 @@ class NiconicoDmcFD(FileDownloader):
heartbeat_data = heartbeat_info_dict['data'].encode()
heartbeat_interval = heartbeat_info_dict.get('interval', 30)
- request = sanitized_Request(heartbeat_url, heartbeat_data)
+ request = Request(heartbeat_url, heartbeat_data)
def heartbeat():
try:
diff --git a/yt_dlp/downloader/youtube_live_chat.py b/yt_dlp/downloader/youtube_live_chat.py
index 5928fecf0..c7a86374a 100644
--- a/yt_dlp/downloader/youtube_live_chat.py
+++ b/yt_dlp/downloader/youtube_live_chat.py
@@ -1,8 +1,8 @@
import json
import time
-import urllib.error
from .fragment import FragmentFD
+from ..networking.exceptions import HTTPError
from ..utils import (
RegexNotFoundError,
RetryManager,
@@ -10,6 +10,7 @@ from ..utils import (
int_or_none,
try_get,
)
+from ..utils.networking import HTTPHeaderDict
class YoutubeLiveChatFD(FragmentFD):
@@ -37,10 +38,7 @@ class YoutubeLiveChatFD(FragmentFD):
start_time = int(time.time() * 1000)
def dl_fragment(url, data=None, headers=None):
- http_headers = info_dict.get('http_headers', {})
- if headers:
- http_headers = http_headers.copy()
- http_headers.update(headers)
+ http_headers = HTTPHeaderDict(info_dict.get('http_headers'), headers)
return self._download_fragment(ctx, url, info_dict, http_headers, data)
def parse_actions_replay(live_chat_continuation):
@@ -129,7 +127,7 @@ class YoutubeLiveChatFD(FragmentFD):
or frag_index == 1 and try_refresh_replay_beginning
or parse_actions_replay)
return (True, *func(live_chat_continuation))
- except urllib.error.HTTPError as err:
+ except HTTPError as err:
retry.error = err
continue
return False, None, None, None
diff --git a/yt_dlp/extractor/abematv.py b/yt_dlp/extractor/abematv.py
index c9166b6b8..98ece8da7 100644
--- a/yt_dlp/extractor/abematv.py
+++ b/yt_dlp/extractor/abematv.py
@@ -22,7 +22,6 @@ from ..utils import (
int_or_none,
intlist_to_bytes,
OnDemandPagedList,
- request_to_url,
time_seconds,
traverse_obj,
update_url_query,
@@ -137,7 +136,7 @@ class AbemaLicenseHandler(urllib.request.BaseHandler):
return intlist_to_bytes(aes_ecb_decrypt(encvideokey, enckey))
def abematv_license_open(self, url):
- url = request_to_url(url)
+ url = url.get_full_url() if isinstance(url, urllib.request.Request) else url
ticket = urllib.parse.urlparse(url).netloc
response_data = self._get_videokey_from_ticket(ticket)
return urllib.response.addinfourl(io.BytesIO(response_data), headers={
diff --git a/yt_dlp/extractor/adn.py b/yt_dlp/extractor/adn.py
index f1f55e87f..b59dbc850 100644
--- a/yt_dlp/extractor/adn.py
+++ b/yt_dlp/extractor/adn.py
@@ -6,10 +6,8 @@ import random
from .common import InfoExtractor
from ..aes import aes_cbc_decrypt_bytes, unpad_pkcs7
-from ..compat import (
- compat_HTTPError,
- compat_b64decode,
-)
+from ..compat import compat_b64decode
+from ..networking.exceptions import HTTPError
from ..utils import (
ass_subtitles_timecode,
bytes_to_intlist,
@@ -142,9 +140,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
self._HEADERS = {'authorization': 'Bearer ' + access_token}
except ExtractorError as e:
message = None
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
resp = self._parse_json(
- e.cause.read().decode(), None, fatal=False) or {}
+ e.cause.response.read().decode(), None, fatal=False) or {}
message = resp.get('message') or resp.get('code')
self.report_warning(message or self._LOGIN_ERR_MESSAGE)
@@ -195,14 +193,14 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
})
break
except ExtractorError as e:
- if not isinstance(e.cause, compat_HTTPError):
+ if not isinstance(e.cause, HTTPError):
raise e
- if e.cause.code == 401:
+ if e.cause.status == 401:
# This usually goes away with a different random pkcs1pad, so retry
continue
- error = self._parse_json(e.cause.read(), video_id)
+ error = self._parse_json(e.cause.response.read(), video_id)
message = error.get('message')
if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
self.raise_geo_restricted(msg=message)
diff --git a/yt_dlp/extractor/adobepass.py b/yt_dlp/extractor/adobepass.py
index 722a534ed..5eed0ca22 100644
--- a/yt_dlp/extractor/adobepass.py
+++ b/yt_dlp/extractor/adobepass.py
@@ -2,11 +2,11 @@ import getpass
import json
import re
import time
-import urllib.error
import xml.etree.ElementTree as etree
from .common import InfoExtractor
from ..compat import compat_urlparse
+from ..networking.exceptions import HTTPError
from ..utils import (
NO_DEFAULT,
ExtractorError,
@@ -1394,7 +1394,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
form_page, urlh = form_page_res
post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
if not re.match(r'https?://', post_url):
- post_url = compat_urlparse.urljoin(urlh.geturl(), post_url)
+ post_url = compat_urlparse.urljoin(urlh.url, post_url)
form_data = self._hidden_inputs(form_page)
form_data.update(data)
return self._download_webpage_handle(
@@ -1619,7 +1619,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
hidden_data['history'] = 1
provider_login_page_res = self._download_webpage_handle(
- urlh.geturl(), video_id, 'Sending first bookend',
+ urlh.url, video_id, 'Sending first bookend',
query=hidden_data)
provider_association_redirect, urlh = post_form(
@@ -1629,7 +1629,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
})
provider_refresh_redirect_url = extract_redirect_url(
- provider_association_redirect, url=urlh.geturl())
+ provider_association_redirect, url=urlh.url)
last_bookend_page, urlh = self._download_webpage_handle(
provider_refresh_redirect_url, video_id,
@@ -1638,7 +1638,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
hidden_data['history'] = 3
mvpd_confirm_page_res = self._download_webpage_handle(
- urlh.geturl(), video_id, 'Sending final bookend',
+ urlh.url, video_id, 'Sending final bookend',
query=hidden_data)
post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1652,7 +1652,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
hidden_data['history_val'] = 1
provider_login_redirect_page_res = self._download_webpage_handle(
- urlh.geturl(), video_id, 'Sending First Bookend',
+ urlh.url, video_id, 'Sending First Bookend',
query=hidden_data)
provider_login_redirect_page, urlh = provider_login_redirect_page_res
@@ -1680,7 +1680,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
})
provider_refresh_redirect_url = extract_redirect_url(
- provider_association_redirect, url=urlh.geturl())
+ provider_association_redirect, url=urlh.url)
last_bookend_page, urlh = self._download_webpage_handle(
provider_refresh_redirect_url, video_id,
@@ -1690,7 +1690,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
hidden_data['history_val'] = 3
mvpd_confirm_page_res = self._download_webpage_handle(
- urlh.geturl(), video_id, 'Sending Final Bookend',
+ urlh.url, video_id, 'Sending Final Bookend',
query=hidden_data)
post_form(mvpd_confirm_page_res, 'Confirming Login')
@@ -1699,7 +1699,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
# based redirect that should be followed.
provider_redirect_page, urlh = provider_redirect_page_res
provider_refresh_redirect_url = extract_redirect_url(
- provider_redirect_page, url=urlh.geturl())
+ provider_redirect_page, url=urlh.url)
if provider_refresh_redirect_url:
provider_redirect_page_res = self._download_webpage_handle(
provider_refresh_redirect_url, video_id,
@@ -1724,7 +1724,7 @@ class AdobePassIE(InfoExtractor): # XXX: Conventionally, base classes should en
'requestor_id': requestor_id,
}), headers=mvpd_headers)
except ExtractorError as e:
- if not mso_id and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+ if not mso_id and isinstance(e.cause, HTTPError) and e.cause.status == 401:
raise_mvpd_required()
raise
if '<pendingLogout' in session:
diff --git a/yt_dlp/extractor/ant1newsgr.py b/yt_dlp/extractor/ant1newsgr.py
index 7b384b22d..217e3acc4 100644
--- a/yt_dlp/extractor/ant1newsgr.py
+++ b/yt_dlp/extractor/ant1newsgr.py
@@ -1,8 +1,8 @@
import urllib.parse
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
ExtractorError,
determine_ext,
scale_thumbnails_to_max_format_width,
@@ -121,7 +121,7 @@ class Ant1NewsGrEmbedIE(Ant1NewsGrBaseIE):
canonical_url = self._request_webpage(
HEADRequest(url), video_id,
note='Resolve canonical player URL',
- errnote='Could not resolve canonical player URL').geturl()
+ errnote='Could not resolve canonical player URL').url
_, netloc, _, _, query, _ = urllib.parse.urlparse(canonical_url)
cid = urllib.parse.parse_qs(query)['cid'][0]
diff --git a/yt_dlp/extractor/archiveorg.py b/yt_dlp/extractor/archiveorg.py
index 4ccd39825..2541cd6fd 100644
--- a/yt_dlp/extractor/archiveorg.py
+++ b/yt_dlp/extractor/archiveorg.py
@@ -1,16 +1,16 @@
import json
import re
-import urllib.error
import urllib.parse
from .common import InfoExtractor
from .naver import NaverBaseIE
from .youtube import YoutubeBaseInfoExtractor, YoutubeIE
-from ..compat import compat_HTTPError, compat_urllib_parse_unquote
+from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
from ..utils import (
KNOWN_EXTENSIONS,
ExtractorError,
- HEADRequest,
bug_reports_message,
clean_html,
dict_get,
@@ -899,7 +899,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
video_id, note='Fetching archived video file url', expected_status=True)
except ExtractorError as e:
# HTTP Error 404 is expected if the video is not saved.
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 404:
self.raise_no_formats(
'The requested video is not archived, indexed, or there is an issue with web.archive.org (try again later)', expected=True)
else:
@@ -926,7 +926,7 @@ class YoutubeWebArchiveIE(InfoExtractor):
info['thumbnails'] = self._extract_thumbnails(video_id)
if urlh:
- url = compat_urllib_parse_unquote(urlh.geturl())
+ url = compat_urllib_parse_unquote(urlh.url)
video_file_url_qs = parse_qs(url)
# Attempt to recover any ext & format info from playback url & response headers
format = {'url': url, 'filesize': int_or_none(urlh.headers.get('x-archive-orig-content-length'))}
@@ -1052,7 +1052,7 @@ class VLiveWebArchiveIE(InfoExtractor):
try:
return self._download_webpage(f'https://web.archive.org/web/{timestamp}id_/{url}', video_id, **kwargs)
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 404:
raise ExtractorError('Page was not archived', expected=True)
retry.error = e
continue
diff --git a/yt_dlp/extractor/atresplayer.py b/yt_dlp/extractor/atresplayer.py
index a20e7f988..3a44e5265 100644
--- a/yt_dlp/extractor/atresplayer.py
+++ b/yt_dlp/extractor/atresplayer.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -34,8 +34,8 @@ class AtresPlayerIE(InfoExtractor):
_API_BASE = 'https://api.atresplayer.com/'
def _handle_error(self, e, code):
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == code:
- error = self._parse_json(e.cause.read(), None)
+ if isinstance(e.cause, HTTPError) and e.cause.status == code:
+ error = self._parse_json(e.cause.response.read(), None)
if error.get('error') == 'required_registered':
self.raise_login_required()
raise ExtractorError(error['error_description'], expected=True)
diff --git a/yt_dlp/extractor/bbc.py b/yt_dlp/extractor/bbc.py
index 9d28e70a3..a55cdef2b 100644
--- a/yt_dlp/extractor/bbc.py
+++ b/yt_dlp/extractor/bbc.py
@@ -2,11 +2,11 @@ import functools
import itertools
import json
import re
-import urllib.error
import xml.etree.ElementTree
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import compat_str, compat_urlparse
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
OnDemandPagedList,
@@ -277,7 +277,7 @@ class BBCCoUkIE(InfoExtractor):
post_url, None, 'Logging in', data=urlencode_postdata(login_form),
headers={'Referer': self._LOGIN_URL})
- if self._LOGIN_URL in urlh.geturl():
+ if self._LOGIN_URL in urlh.url:
error = clean_html(get_element_by_class('form-message', response))
if error:
raise ExtractorError(
@@ -388,8 +388,8 @@ class BBCCoUkIE(InfoExtractor):
href, programme_id, ext='mp4', entry_protocol='m3u8_native',
m3u8_id=format_id, fatal=False)
except ExtractorError as e:
- if not (isinstance(e.exc_info[1], urllib.error.HTTPError)
- and e.exc_info[1].code in (403, 404)):
+ if not (isinstance(e.exc_info[1], HTTPError)
+ and e.exc_info[1].status in (403, 404)):
raise
fmts = []
formats.extend(fmts)
@@ -472,7 +472,7 @@ class BBCCoUkIE(InfoExtractor):
return programme_id, title, description, duration, formats, subtitles
except ExtractorError as ee:
- if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
+ if not (isinstance(ee.cause, HTTPError) and ee.cause.status == 404):
raise
# fallback to legacy playlist
@@ -983,7 +983,7 @@ class BBCIE(BBCCoUkIE): # XXX: Do not subclass from concrete IE
# Some playlist URL may fail with 500, at the same time
# the other one may work fine (e.g.
# http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 500:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 500:
continue
raise
if entry:
diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index e8714a33a..cb7ab2a17 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -4,11 +4,11 @@ import hashlib
import itertools
import math
import time
-import urllib.error
import urllib.parse
from .common import InfoExtractor, SearchInfoExtractor
from ..dependencies import Cryptodome
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
GeoRestrictedError,
@@ -614,7 +614,7 @@ class BilibiliSpaceVideoIE(BilibiliSpaceBaseIE):
response = self._download_json('https://api.bilibili.com/x/space/wbi/arc/search',
playlist_id, note=f'Downloading page {page_idx}', query=query)
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 412:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 412:
raise ExtractorError(
'Request is blocked by server (412), please add cookies, wait and try later.', expected=True)
raise
diff --git a/yt_dlp/extractor/bitchute.py b/yt_dlp/extractor/bitchute.py
index a6779505e..0805b8b46 100644
--- a/yt_dlp/extractor/bitchute.py
+++ b/yt_dlp/extractor/bitchute.py
@@ -2,9 +2,9 @@ import functools
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
OnDemandPagedList,
clean_html,
get_element_by_class,
diff --git a/yt_dlp/extractor/bravotv.py b/yt_dlp/extractor/bravotv.py
index 13cc1927f..419fe8c9c 100644
--- a/yt_dlp/extractor/bravotv.py
+++ b/yt_dlp/extractor/bravotv.py
@@ -1,6 +1,6 @@
from .adobepass import AdobePassIE
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
extract_attributes,
float_or_none,
get_element_html_by_class,
@@ -155,7 +155,7 @@ class BravoTVIE(AdobePassIE):
chapters = None
m3u8_url = self._request_webpage(HEADRequest(
- update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').geturl()
+ update_url_query(f'{tp_url}/stream.m3u8', query)), video_id, 'Checking m3u8 URL').url
if 'mpeg_cenc' in m3u8_url:
self.report_drm(video_id)
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
diff --git a/yt_dlp/extractor/brightcove.py b/yt_dlp/extractor/brightcove.py
index cd0e8ff27..61b18412d 100644
--- a/yt_dlp/extractor/brightcove.py
+++ b/yt_dlp/extractor/brightcove.py
@@ -7,10 +7,10 @@ from .adobepass import AdobePassIE
from .common import InfoExtractor
from ..compat import (
compat_etree_fromstring,
- compat_HTTPError,
compat_parse_qs,
compat_urlparse,
)
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
dict_get,
@@ -915,8 +915,8 @@ class BrightcoveNewIE(BrightcoveNewBaseIE):
json_data = self._download_json(api_url, video_id, headers=headers)
break
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
- json_data = self._parse_json(e.cause.read().decode(), video_id)[0]
+ if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
+ json_data = self._parse_json(e.cause.response.read().decode(), video_id)[0]
message = json_data.get('message') or json_data['error_code']
if json_data.get('error_subcode') == 'CLIENT_GEO':
self.raise_geo_restricted(msg=message)
diff --git a/yt_dlp/extractor/canalplus.py b/yt_dlp/extractor/canalplus.py
index b7e2f9dd4..3ff5c3fbf 100644
--- a/yt_dlp/extractor/canalplus.py
+++ b/yt_dlp/extractor/canalplus.py
@@ -64,7 +64,7 @@ class CanalplusIE(InfoExtractor):
# response = self._request_webpage(
# HEADRequest(fmt_url), video_id,
# 'Checking if the video is georestricted')
- # if '/blocage' in response.geturl():
+ # if '/blocage' in response.url:
# raise ExtractorError(
# 'The video is not available in your country',
# expected=True)
diff --git a/yt_dlp/extractor/cbsnews.py b/yt_dlp/extractor/cbsnews.py
index 65ecc62f0..5a8ebb847 100644
--- a/yt_dlp/extractor/cbsnews.py
+++ b/yt_dlp/extractor/cbsnews.py
@@ -7,9 +7,9 @@ import zlib
from .anvato import AnvatoIE
from .common import InfoExtractor
from .paramountplus import ParamountPlusIE
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
UserNotLive,
determine_ext,
float_or_none,
diff --git a/yt_dlp/extractor/ceskatelevize.py b/yt_dlp/extractor/ceskatelevize.py
index be2b0bb43..8390160a0 100644
--- a/yt_dlp/extractor/ceskatelevize.py
+++ b/yt_dlp/extractor/ceskatelevize.py
@@ -1,20 +1,20 @@
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse_unquote,
- compat_urllib_parse_urlparse,
-)
+from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse
+from ..networking import Request
from ..utils import (
ExtractorError,
float_or_none,
- sanitized_Request,
str_or_none,
traverse_obj,
urlencode_postdata,
- USER_AGENTS,
)
+USER_AGENTS = {
+ 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
+}
+
class CeskaTelevizeIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady|zive)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
@@ -97,7 +97,7 @@ class CeskaTelevizeIE(InfoExtractor):
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage, urlh = self._download_webpage_handle(url, playlist_id)
- parsed_url = compat_urllib_parse_urlparse(urlh.geturl())
+ parsed_url = compat_urllib_parse_urlparse(urlh.url)
site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
playlist_title = self._og_search_title(webpage, default=None)
if site_name and playlist_title:
@@ -163,16 +163,16 @@ class CeskaTelevizeIE(InfoExtractor):
entries = []
for user_agent in (None, USER_AGENTS['Safari']):
- req = sanitized_Request(
+ req = Request(
'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
data=urlencode_postdata(data))
- req.add_header('Content-type', 'application/x-www-form-urlencoded')
- req.add_header('x-addr', '127.0.0.1')
- req.add_header('X-Requested-With', 'XMLHttpRequest')
+ req.headers['Content-type'] = 'application/x-www-form-urlencoded'
+ req.headers['x-addr'] = '127.0.0.1'
+ req.headers['X-Requested-With'] = 'XMLHttpRequest'
if user_agent:
- req.add_header('User-Agent', user_agent)
- req.add_header('Referer', url)
+ req.headers['User-Agent'] = user_agent
+ req.headers['Referer'] = url
playlistpage = self._download_json(req, playlist_id, fatal=False)
@@ -183,8 +183,8 @@ class CeskaTelevizeIE(InfoExtractor):
if playlist_url == 'error_region':
raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
- req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
- req.add_header('Referer', url)
+ req = Request(compat_urllib_parse_unquote(playlist_url))
+ req.headers['Referer'] = url
playlist = self._download_json(req, playlist_id, fatal=False)
if not playlist:
diff --git a/yt_dlp/extractor/cinetecamilano.py b/yt_dlp/extractor/cinetecamilano.py
index 5e770ebac..9cffa11e8 100644
--- a/yt_dlp/extractor/cinetecamilano.py
+++ b/yt_dlp/extractor/cinetecamilano.py
@@ -1,6 +1,6 @@
import json
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
float_or_none,
@@ -40,7 +40,7 @@ class CinetecaMilanoIE(InfoExtractor):
'Authorization': try_get(self._get_cookies('https://www.cinetecamilano.it'), lambda x: f'Bearer {x["cnt-token"].value}') or ''
})
except ExtractorError as e:
- if ((isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 500)
+ if ((isinstance(e.cause, HTTPError) and e.cause.status == 500)
or isinstance(e.cause, json.JSONDecodeError)):
self.raise_login_required(method='cookies')
raise
diff --git a/yt_dlp/extractor/ciscowebex.py b/yt_dlp/extractor/ciscowebex.py
index 40430505d..85585dffb 100644
--- a/yt_dlp/extractor/ciscowebex.py
+++ b/yt_dlp/extractor/ciscowebex.py
@@ -33,7 +33,7 @@ class CiscoWebexIE(InfoExtractor):
if rcid:
webpage = self._download_webpage(url, None, note='Getting video ID')
url = self._search_regex(self._VALID_URL, webpage, 'redirection url', group='url')
- url = self._request_webpage(url, None, note='Resolving final URL').geturl()
+ url = self._request_webpage(url, None, note='Resolving final URL').url
mobj = self._match_valid_url(url)
subdomain = mobj.group('subdomain')
siteurl = mobj.group('siteurl_1') or mobj.group('siteurl_2')
@@ -49,7 +49,7 @@ class CiscoWebexIE(InfoExtractor):
'https://%s.webex.com/webappng/api/v1/recordings/%s/stream' % (subdomain, video_id),
video_id, headers=headers, query={'siteurl': siteurl}, expected_status=(403, 429))
- if urlh.getcode() == 403:
+ if urlh.status == 403:
if stream['code'] == 53004:
self.raise_login_required()
if stream['code'] == 53005:
@@ -59,7 +59,7 @@ class CiscoWebexIE(InfoExtractor):
'This video is protected by a password, use the --video-password option', expected=True)
raise ExtractorError(f'{self.IE_NAME} said: {stream["code"]} - {stream["message"]}', expected=True)
- if urlh.getcode() == 429:
+ if urlh.status == 429:
self.raise_login_required(
f'{self.IE_NAME} asks you to solve a CAPTCHA. Solve CAPTCHA in browser and',
method='cookies')
diff --git a/yt_dlp/extractor/common.py b/yt_dlp/extractor/common.py
index 63156d3ac..d44918776 100644
--- a/yt_dlp/extractor/common.py
+++ b/yt_dlp/extractor/common.py
@@ -31,8 +31,12 @@ from ..compat import (
from ..cookies import LenientSimpleCookie
from ..downloader.f4m import get_base_url, remove_encrypted_media
from ..downloader.hls import HlsFD
-from ..networking.common import HEADRequest, Request
-from ..networking.exceptions import network_exceptions
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import (
+ HTTPError,
+ IncompleteRead,
+ network_exceptions,
+)
from ..utils import (
IDENTITY,
JSON_LD_RE,
@@ -729,7 +733,7 @@ class InfoExtractor:
e.ie = e.ie or self.IE_NAME,
e.traceback = e.traceback or sys.exc_info()[2]
raise
- except http.client.IncompleteRead as e:
+ except IncompleteRead as e:
raise ExtractorError('A network error has occurred.', cause=e, expected=True, video_id=self.get_temp_id(url))
except (KeyError, StopIteration) as e:
raise ExtractorError('An extractor error has occurred.', cause=e, video_id=self.get_temp_id(url))
@@ -788,16 +792,19 @@ class InfoExtractor:
@staticmethod
def __can_accept_status_code(err, expected_status):
- assert isinstance(err, urllib.error.HTTPError)
+ assert isinstance(err, HTTPError)
if expected_status is None:
return False
elif callable(expected_status):
- return expected_status(err.code) is True
+ return expected_status(err.status) is True
else:
- return err.code in variadic(expected_status)
+ return err.status in variadic(expected_status)
def _create_request(self, url_or_request, data=None, headers=None, query=None):
if isinstance(url_or_request, urllib.request.Request):
+ self._downloader.deprecation_warning(
+ 'Passing a urllib.request.Request to _create_request() is deprecated. '
+ 'Use yt_dlp.networking.common.Request instead.')
url_or_request = urllib_req_to_req(url_or_request)
elif not isinstance(url_or_request, Request):
url_or_request = Request(url_or_request)
@@ -839,7 +846,7 @@ class InfoExtractor:
try:
return self._downloader.urlopen(self._create_request(url_or_request, data, headers, query))
except network_exceptions as err:
- if isinstance(err, urllib.error.HTTPError):
+ if isinstance(err, HTTPError):
if self.__can_accept_status_code(err, expected_status):
return err.response
@@ -973,11 +980,11 @@ class InfoExtractor:
if prefix is not None:
webpage_bytes = prefix + webpage_bytes
if self.get_param('dump_intermediate_pages', False):
- self.to_screen('Dumping request to ' + urlh.geturl())
+ self.to_screen('Dumping request to ' + urlh.url)
dump = base64.b64encode(webpage_bytes).decode('ascii')
self._downloader.to_screen(dump)
if self.get_param('write_pages'):
- filename = self._request_dump_filename(urlh.geturl(), video_id)
+ filename = self._request_dump_filename(urlh.url, video_id)
self.to_screen(f'Saving request to {filename}')
with open(filename, 'wb') as outf:
outf.write(webpage_bytes)
@@ -1109,7 +1116,7 @@ class InfoExtractor:
while True:
try:
return self.__download_webpage(url_or_request, video_id, note, errnote, None, fatal, *args, **kwargs)
- except http.client.IncompleteRead as e:
+ except IncompleteRead as e:
try_count += 1
if try_count >= tries:
raise e
@@ -1806,7 +1813,7 @@ class InfoExtractor:
return []
manifest, urlh = res
- manifest_url = urlh.geturl()
+ manifest_url = urlh.url
return self._parse_f4m_formats(
manifest, manifest_url, video_id, preference=preference, quality=quality, f4m_id=f4m_id,
@@ -1965,7 +1972,7 @@ class InfoExtractor:
return [], {}
m3u8_doc, urlh = res
- m3u8_url = urlh.geturl()
+ m3u8_url = urlh.url
return self._parse_m3u8_formats_and_subtitles(
m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
@@ -2243,7 +2250,7 @@ class InfoExtractor:
return [], {}
smil, urlh = res
- smil_url = urlh.geturl()
+ smil_url = urlh.url
namespace = self._parse_smil_namespace(smil)
@@ -2266,7 +2273,7 @@ class InfoExtractor:
return {}
smil, urlh = res
- smil_url = urlh.geturl()
+ smil_url = urlh.url
return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
@@ -2458,7 +2465,7 @@ class InfoExtractor:
return []
xspf, urlh = res
- xspf_url = urlh.geturl()
+ xspf_url = urlh.url
return self._parse_xspf(
xspf, playlist_id, xspf_url=xspf_url,
@@ -2529,7 +2536,7 @@ class InfoExtractor:
return [], {}
# We could have been redirected to a new url when we retrieved our mpd file.
- mpd_url = urlh.geturl()
+ mpd_url = urlh.url
mpd_base_url = base_url(mpd_url)
return self._parse_mpd_formats_and_subtitles(
@@ -2900,7 +2907,7 @@ class InfoExtractor:
if ism_doc is None:
return [], {}
- return self._parse_ism_formats_and_subtitles(ism_doc, urlh.geturl(), ism_id)
+ return self._parse_ism_formats_and_subtitles(ism_doc, urlh.url, ism_id)
def _parse_ism_formats_and_subtitles(self, ism_doc, ism_url, ism_id=None):
"""
diff --git a/yt_dlp/extractor/crackle.py b/yt_dlp/extractor/crackle.py
index 46100151a..1ef90b5a0 100644
--- a/yt_dlp/extractor/crackle.py
+++ b/yt_dlp/extractor/crackle.py
@@ -4,7 +4,7 @@ import re
import time
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
float_or_none,
@@ -113,7 +113,7 @@ class CrackleIE(InfoExtractor):
errnote='Unable to download media JSON')
except ExtractorError as e:
# 401 means geo restriction, trying next country
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
continue
raise
diff --git a/yt_dlp/extractor/crunchyroll.py b/yt_dlp/extractor/crunchyroll.py
index 910504ed2..adb3d5dcf 100644
--- a/yt_dlp/extractor/crunchyroll.py
+++ b/yt_dlp/extractor/crunchyroll.py
@@ -1,7 +1,7 @@
import base64
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
float_or_none,
@@ -114,7 +114,7 @@ class CrunchyrollBaseIE(InfoExtractor):
result = self._call_base_api(
path, internal_id, lang, f'Downloading {note} JSON ({self._API_ENDPOINT})', query=query)
except ExtractorError as error:
- if isinstance(error.cause, urllib.error.HTTPError) and error.cause.code == 404:
+ if isinstance(error.cause, HTTPError) and error.cause.status == 404:
return None
raise
diff --git a/yt_dlp/extractor/cultureunplugged.py b/yt_dlp/extractor/cultureunplugged.py
index 2fb22800f..9c8509f1f 100644
--- a/yt_dlp/extractor/cultureunplugged.py
+++ b/yt_dlp/extractor/cultureunplugged.py
@@ -1,10 +1,8 @@
import time
from .common import InfoExtractor
-from ..utils import (
- int_or_none,
- HEADRequest,
-)
+from ..networking import HEADRequest
+from ..utils import int_or_none
class CultureUnpluggedIE(InfoExtractor):
diff --git a/yt_dlp/extractor/dacast.py b/yt_dlp/extractor/dacast.py
index cf683bad4..4e81aa4a7 100644
--- a/yt_dlp/extractor/dacast.py
+++ b/yt_dlp/extractor/dacast.py
@@ -1,9 +1,9 @@
import hashlib
import re
import time
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
classproperty,
@@ -105,7 +105,7 @@ class DacastVODIE(DacastBaseIE):
formats = self._extract_m3u8_formats(hls_url, video_id, 'mp4', m3u8_id='hls')
except ExtractorError as e:
# CDN will randomly respond with 403
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
retry.error = e
continue
raise
diff --git a/yt_dlp/extractor/dailymotion.py b/yt_dlp/extractor/dailymotion.py
index 2a44718fb..21263d41b 100644
--- a/yt_dlp/extractor/dailymotion.py
+++ b/yt_dlp/extractor/dailymotion.py
@@ -3,7 +3,7 @@ import json
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
OnDemandPagedList,
@@ -68,9 +68,9 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
None, 'Downloading Access Token',
data=urlencode_postdata(data))['access_token']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
raise ExtractorError(self._parse_json(
- e.cause.read().decode(), xid)['error_description'], expected=True)
+ e.cause.response.read().decode(), xid)['error_description'], expected=True)
raise
self._set_dailymotion_cookie('access_token' if username else 'client_token', token)
self._HEADERS['Authorization'] = 'Bearer ' + token
diff --git a/yt_dlp/extractor/discovery.py b/yt_dlp/extractor/discovery.py
index e6e109d5c..75b464353 100644
--- a/yt_dlp/extractor/discovery.py
+++ b/yt_dlp/extractor/discovery.py
@@ -3,8 +3,8 @@ import string
from .discoverygo import DiscoveryGoBaseIE
from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
from ..utils import ExtractorError
-from ..compat import compat_HTTPError
class DiscoveryIE(DiscoveryGoBaseIE):
@@ -100,9 +100,9 @@ class DiscoveryIE(DiscoveryGoBaseIE):
self._API_BASE_URL + 'streaming/video/' + video_id,
display_id, 'Downloading streaming JSON metadata', headers=headers)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+ if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
e_description = self._parse_json(
- e.cause.read().decode(), display_id)['description']
+ e.cause.response.read().decode(), display_id)['description']
if 'resource not available for country' in e_description:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
if 'Authorized Networks' in e_description:
diff --git a/yt_dlp/extractor/dplay.py b/yt_dlp/extractor/dplay.py
index cf6d14934..6404752f7 100644
--- a/yt_dlp/extractor/dplay.py
+++ b/yt_dlp/extractor/dplay.py
@@ -2,7 +2,7 @@ import json
import uuid
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -39,7 +39,7 @@ class DPlayBaseIE(InfoExtractor):
return f'Bearer {token}'
def _process_errors(self, e, geo_countries):
- info = self._parse_json(e.cause.read().decode('utf-8'), None)
+ info = self._parse_json(e.cause.response.read().decode('utf-8'), None)
error = info['errors'][0]
error_code = error.get('code')
if error_code == 'access.denied.geoblocked':
@@ -87,7 +87,7 @@ class DPlayBaseIE(InfoExtractor):
'include': 'images,primaryChannel,show,tags'
})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
self._process_errors(e, geo_countries)
raise
video_id = video['data']['id']
@@ -99,7 +99,7 @@ class DPlayBaseIE(InfoExtractor):
streaming = self._download_video_playback_info(
disco_base, video_id, headers)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
self._process_errors(e, geo_countries)
raise
for format_dict in streaming:
diff --git a/yt_dlp/extractor/eagleplatform.py b/yt_dlp/extractor/eagleplatform.py
index 9ebd24d80..739d17912 100644
--- a/yt_dlp/extractor/eagleplatform.py
+++ b/yt_dlp/extractor/eagleplatform.py
@@ -2,7 +2,7 @@ import functools
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -111,8 +111,8 @@ class EaglePlatformIE(InfoExtractor):
response = super(EaglePlatformIE, self)._download_json(
url_or_request, video_id, *args, **kwargs)
except ExtractorError as ee:
- if isinstance(ee.cause, compat_HTTPError):
- response = self._parse_json(ee.cause.read().decode('utf-8'), video_id)
+ if isinstance(ee.cause, HTTPError):
+ response = self._parse_json(ee.cause.response.read().decode('utf-8'), video_id)
self._handle_error(response)
raise
return response
diff --git a/yt_dlp/extractor/eitb.py b/yt_dlp/extractor/eitb.py
index bd027da6b..66afbb6bb 100644
--- a/yt_dlp/extractor/eitb.py
+++ b/yt_dlp/extractor/eitb.py
@@ -1,10 +1,6 @@
from .common import InfoExtractor
-from ..utils import (
- float_or_none,
- int_or_none,
- parse_iso8601,
- sanitized_Request,
-)
+from ..networking import Request
+from ..utils import float_or_none, int_or_none, parse_iso8601
class EitbIE(InfoExtractor):
@@ -54,7 +50,7 @@ class EitbIE(InfoExtractor):
hls_url = media.get('HLS_SURL')
if hls_url:
- request = sanitized_Request(
+ request = Request(
'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
headers={'Referer': url})
token_data = self._download_json(
diff --git a/yt_dlp/extractor/eporner.py b/yt_dlp/extractor/eporner.py
index a2337979b..aee2dee58 100644
--- a/yt_dlp/extractor/eporner.py
+++ b/yt_dlp/extractor/eporner.py
@@ -52,7 +52,7 @@ class EpornerIE(InfoExtractor):
webpage, urlh = self._download_webpage_handle(url, display_id)
- video_id = self._match_id(urlh.geturl())
+ video_id = self._match_id(urlh.url)
hash = self._search_regex(
r'hash\s*[:=]\s*["\']([\da-f]{32})', webpage, 'hash')
diff --git a/yt_dlp/extractor/facebook.py b/yt_dlp/extractor/facebook.py
index 9d871eb28..9f4d3fb78 100644
--- a/yt_dlp/extractor/facebook.py
+++ b/yt_dlp/extractor/facebook.py
@@ -8,6 +8,8 @@ from ..compat import (
compat_str,
compat_urllib_parse_unquote,
)
+from ..networking import Request
+from ..networking.exceptions import network_exceptions
from ..utils import (
ExtractorError,
clean_html,
@@ -19,11 +21,9 @@ from ..utils import (
int_or_none,
js_to_json,
merge_dicts,
- network_exceptions,
parse_count,
parse_qs,
qualities,
- sanitized_Request,
traverse_obj,
try_get,
url_or_none,
@@ -319,7 +319,7 @@ class FacebookIE(InfoExtractor):
}
def _perform_login(self, username, password):
- login_page_req = sanitized_Request(self._LOGIN_URL)
+ login_page_req = Request(self._LOGIN_URL)
self._set_cookie('facebook.com', 'locale', 'en_US')
login_page = self._download_webpage(login_page_req, None,
note='Downloading login page',
@@ -340,8 +340,8 @@ class FacebookIE(InfoExtractor):
'timezone': '-60',
'trynum': '1',
}
- request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
- request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ request = Request(self._LOGIN_URL, urlencode_postdata(login_form))
+ request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
try:
login_results = self._download_webpage(request, None,
note='Logging in', errnote='unable to fetch login page')
@@ -367,8 +367,8 @@ class FacebookIE(InfoExtractor):
'h': h,
'name_action_selected': 'dont_save',
}
- check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
- check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ check_req = Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+ check_req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
check_response = self._download_webpage(check_req, None,
note='Confirming login')
if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
diff --git a/yt_dlp/extractor/fc2.py b/yt_dlp/extractor/fc2.py
index dd5e088fc..ba19b6cab 100644
--- a/yt_dlp/extractor/fc2.py
+++ b/yt_dlp/extractor/fc2.py
@@ -3,11 +3,11 @@ import re
from .common import InfoExtractor
from ..compat import compat_parse_qs
from ..dependencies import websockets
+from ..networking import Request
from ..utils import (
ExtractorError,
WebSocketsWrapper,
js_to_json,
- sanitized_Request,
traverse_obj,
update_url_query,
urlencode_postdata,
@@ -57,7 +57,7 @@ class FC2IE(InfoExtractor):
}
login_data = urlencode_postdata(login_form_strs)
- request = sanitized_Request(
+ request = Request(
'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +66,7 @@ class FC2IE(InfoExtractor):
return False
# this is also needed
- login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
+ login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
self._download_webpage(
login_redir, None, note='Login redirect', errnote='Login redirect failed')
diff --git a/yt_dlp/extractor/filmon.py b/yt_dlp/extractor/filmon.py
index 9a93cb984..0cd18f494 100644
--- a/yt_dlp/extractor/filmon.py
+++ b/yt_dlp/extractor/filmon.py
@@ -1,8 +1,6 @@
from .common import InfoExtractor
-from ..compat import (
- compat_str,
- compat_HTTPError,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
qualities,
strip_or_none,
@@ -40,8 +38,8 @@ class FilmOnIE(InfoExtractor):
'https://www.filmon.com/api/vod/movie?id=%s' % video_id,
video_id)['response']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
- errmsg = self._parse_json(e.cause.read().decode(), video_id)['reason']
+ if isinstance(e.cause, HTTPError):
+ errmsg = self._parse_json(e.cause.response.read().decode(), video_id)['reason']
raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
raise
@@ -124,8 +122,8 @@ class FilmOnChannelIE(InfoExtractor):
channel_data = self._download_json(
'http://www.filmon.com/api-v2/channel/' + channel_id, channel_id)['data']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
- errmsg = self._parse_json(e.cause.read().decode(), channel_id)['message']
+ if isinstance(e.cause, HTTPError):
+ errmsg = self._parse_json(e.cause.response.read().decode(), channel_id)['message']
raise ExtractorError('%s said: %s' % (self.IE_NAME, errmsg), expected=True)
raise
diff --git a/yt_dlp/extractor/fox.py b/yt_dlp/extractor/fox.py
index 15c0c48c1..8fb4ada6b 100644
--- a/yt_dlp/extractor/fox.py
+++ b/yt_dlp/extractor/fox.py
@@ -3,10 +3,10 @@ import uuid
from .common import InfoExtractor
from ..compat import (
- compat_HTTPError,
compat_str,
compat_urllib_parse_unquote,
)
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -68,9 +68,9 @@ class FOXIE(InfoExtractor):
'https://api3.fox.com/v2.0/' + path,
video_id, data=data, headers=headers)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
entitlement_issues = self._parse_json(
- e.cause.read().decode(), video_id)['entitlementIssues']
+ e.cause.response.read().decode(), video_id)['entitlementIssues']
for e in entitlement_issues:
if e.get('errorCode') == 1005:
raise ExtractorError(
@@ -123,8 +123,8 @@ class FOXIE(InfoExtractor):
try:
m3u8_url = self._download_json(release_url, video_id)['playURL']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- error = self._parse_json(e.cause.read().decode(), video_id)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ error = self._parse_json(e.cause.response.read().decode(), video_id)
if error.get('exception') == 'GeoLocationBlocked':
self.raise_geo_restricted(countries=['US'])
raise ExtractorError(error['description'], expected=True)
diff --git a/yt_dlp/extractor/foxsports.py b/yt_dlp/extractor/foxsports.py
index f906a1718..8e89ccf84 100644
--- a/yt_dlp/extractor/foxsports.py
+++ b/yt_dlp/extractor/foxsports.py
@@ -1,6 +1,7 @@
from .common import InfoExtractor
from .uplynk import UplynkPreplayIE
-from ..utils import HEADRequest, float_or_none, make_archive_id, smuggle_url
+from ..networking import HEADRequest
+from ..utils import float_or_none, make_archive_id, smuggle_url
class FoxSportsIE(InfoExtractor):
@@ -35,7 +36,7 @@ class FoxSportsIE(InfoExtractor):
'x-api-key': 'cf289e299efdfa39fb6316f259d1de93',
})
preplay_url = self._request_webpage(
- HEADRequest(data['url']), video_id, 'Fetching preplay URL').geturl()
+ HEADRequest(data['url']), video_id, 'Fetching preplay URL').url
return {
'_type': 'url_transparent',
diff --git a/yt_dlp/extractor/fujitv.py b/yt_dlp/extractor/fujitv.py
index 668bb2743..77e826e2d 100644
--- a/yt_dlp/extractor/fujitv.py
+++ b/yt_dlp/extractor/fujitv.py
@@ -1,5 +1,5 @@
-from ..utils import HEADRequest
from .common import InfoExtractor
+from ..networking import HEADRequest
class FujiTVFODPlus7IE(InfoExtractor):
diff --git a/yt_dlp/extractor/funimation.py b/yt_dlp/extractor/funimation.py
index 47c316664..41de85cc6 100644
--- a/yt_dlp/extractor/funimation.py
+++ b/yt_dlp/extractor/funimation.py
@@ -3,7 +3,7 @@ import re
import string
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
@@ -46,8 +46,8 @@ class FunimationBaseIE(InfoExtractor):
}))
FunimationBaseIE._TOKEN = data['token']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read().decode(), None)['error']
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read().decode(), None)['error']
raise ExtractorError(error, expected=True)
raise
diff --git a/yt_dlp/extractor/gdcvault.py b/yt_dlp/extractor/gdcvault.py
index 2878bbd88..4265feb61 100644
--- a/yt_dlp/extractor/gdcvault.py
+++ b/yt_dlp/extractor/gdcvault.py
@@ -2,13 +2,8 @@ import re
from .common import InfoExtractor
from .kaltura import KalturaIE
-from ..utils import (
- HEADRequest,
- remove_start,
- sanitized_Request,
- smuggle_url,
- urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import remove_start, smuggle_url, urlencode_postdata
class GDCVaultIE(InfoExtractor):
@@ -138,8 +133,8 @@ class GDCVaultIE(InfoExtractor):
'password': password,
}
- request = sanitized_Request(login_url, urlencode_postdata(login_form))
- request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ request = Request(login_url, urlencode_postdata(login_form))
+ request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
self._download_webpage(request, display_id, 'Logging in')
start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
self._download_webpage(logout_url, display_id, 'Logging out')
@@ -163,7 +158,7 @@ class GDCVaultIE(InfoExtractor):
video_url = 'http://www.gdcvault.com' + direct_url
# resolve the url so that we can detect the correct extension
video_url = self._request_webpage(
- HEADRequest(video_url), video_id).geturl()
+ HEADRequest(video_url), video_id).url
return {
'id': video_id,
diff --git a/yt_dlp/extractor/generic.py b/yt_dlp/extractor/generic.py
index 87cf11d6b..8fa4c6221 100644
--- a/yt_dlp/extractor/generic.py
+++ b/yt_dlp/extractor/generic.py
@@ -2431,7 +2431,7 @@ class GenericIE(InfoExtractor):
'Accept-Encoding': 'identity',
**smuggled_data.get('http_headers', {})
})
- new_url = full_response.geturl()
+ new_url = full_response.url
url = urllib.parse.urlparse(url)._replace(scheme=urllib.parse.urlparse(new_url).scheme).geturl()
if new_url != extract_basic_auth(url)[0]:
self.report_following_redirect(new_url)
@@ -2529,12 +2529,12 @@ class GenericIE(InfoExtractor):
return self.playlist_result(
self._parse_xspf(
doc, video_id, xspf_url=url,
- xspf_base_url=full_response.geturl()),
+ xspf_base_url=full_response.url),
video_id)
elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
info_dict['formats'], info_dict['subtitles'] = self._parse_mpd_formats_and_subtitles(
doc,
- mpd_base_url=full_response.geturl().rpartition('/')[0],
+ mpd_base_url=full_response.url.rpartition('/')[0],
mpd_url=url)
self._extra_manifest_info(info_dict, url)
self.report_detected('DASH manifest')
@@ -2572,7 +2572,7 @@ class GenericIE(InfoExtractor):
info_dict = types.MappingProxyType(info_dict) # Prevents accidental mutation
video_id = traverse_obj(info_dict, 'display_id', 'id') or self._generic_id(url)
url, smuggled_data = unsmuggle_url(url, {})
- actual_url = urlh.geturl() if urlh else url
+ actual_url = urlh.url if urlh else url
# Sometimes embedded video player is hidden behind percent encoding
# (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448)
diff --git a/yt_dlp/extractor/globo.py b/yt_dlp/extractor/globo.py
index a7be2cb76..df98f093c 100644
--- a/yt_dlp/extractor/globo.py
+++ b/yt_dlp/extractor/globo.py
@@ -8,8 +8,8 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
)
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
ExtractorError,
float_or_none,
orderedSet,
diff --git a/yt_dlp/extractor/googledrive.py b/yt_dlp/extractor/googledrive.py
index 8a4cd1690..2fdec20f6 100644
--- a/yt_dlp/extractor/googledrive.py
+++ b/yt_dlp/extractor/googledrive.py
@@ -228,7 +228,7 @@ class GoogleDriveIE(InfoExtractor):
# Using original URLs may result in redirect loop due to
# google.com's cookies mistakenly used for googleusercontent.com
# redirect URLs (see #23919).
- 'url': urlh.geturl(),
+ 'url': urlh.url,
'ext': determine_ext(title, 'mp4').lower(),
'format_id': 'source',
'quality': 1,
diff --git a/yt_dlp/extractor/hketv.py b/yt_dlp/extractor/hketv.py
index 10879564f..e026996da 100644
--- a/yt_dlp/extractor/hketv.py
+++ b/yt_dlp/extractor/hketv.py
@@ -126,7 +126,7 @@ class HKETVIE(InfoExtractor):
# If we ever wanted to provide the final resolved URL that
# does not require cookies, albeit with a shorter lifespan:
# urlh = self._downloader.urlopen(file_url)
- # resolved_url = urlh.geturl()
+ # resolved_url = urlh.url
label = fmt.get('label')
h = self._FORMAT_HEIGHTS.get(label)
w = h * width // height if h and width and height else None
diff --git a/yt_dlp/extractor/hotnewhiphop.py b/yt_dlp/extractor/hotnewhiphop.py
index f8570cb86..3007fbb53 100644
--- a/yt_dlp/extractor/hotnewhiphop.py
+++ b/yt_dlp/extractor/hotnewhiphop.py
@@ -1,11 +1,7 @@
from .common import InfoExtractor
from ..compat import compat_b64decode
-from ..utils import (
- ExtractorError,
- HEADRequest,
- sanitized_Request,
- urlencode_postdata,
-)
+from ..networking import HEADRequest, Request
+from ..utils import ExtractorError, urlencode_postdata
class HotNewHipHopIE(InfoExtractor):
@@ -36,9 +32,9 @@ class HotNewHipHopIE(InfoExtractor):
('mediaType', 's'),
('mediaId', video_id),
])
- r = sanitized_Request(
+ r = Request(
'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
- r.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
mkd = self._download_json(
r, video_id, note='Requesting media key',
errnote='Could not download media key')
@@ -50,7 +46,7 @@ class HotNewHipHopIE(InfoExtractor):
req = self._request_webpage(
redirect_req, video_id,
note='Resolving final URL', errnote='Could not resolve final URL')
- video_url = req.geturl()
+ video_url = req.url
if video_url.endswith('.html'):
raise ExtractorError('Redirect failed')
diff --git a/yt_dlp/extractor/hotstar.py b/yt_dlp/extractor/hotstar.py
index 591e23b8a..324e9f51d 100644
--- a/yt_dlp/extractor/hotstar.py
+++ b/yt_dlp/extractor/hotstar.py
@@ -6,7 +6,8 @@ import time
import uuid
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
@@ -233,7 +234,7 @@ class HotStarIE(HotStarBaseIE):
'height': int_or_none(playback_set.get('height')),
}]
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
geo_restricted = True
continue
diff --git a/yt_dlp/extractor/hrti.py b/yt_dlp/extractor/hrti.py
index cfec80d14..57b76e46b 100644
--- a/yt_dlp/extractor/hrti.py
+++ b/yt_dlp/extractor/hrti.py
@@ -1,13 +1,13 @@
import json
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking import Request
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
ExtractorError,
int_or_none,
parse_age_limit,
- sanitized_Request,
try_get,
)
@@ -42,7 +42,7 @@ class HRTiBaseIE(InfoExtractor):
'application_version': self._APP_VERSION
}
- req = sanitized_Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
+ req = Request(self._API_URL, data=json.dumps(app_data).encode('utf-8'))
req.get_method = lambda: 'PUT'
resources = self._download_json(
@@ -73,8 +73,8 @@ class HRTiBaseIE(InfoExtractor):
self._login_url, None, note='Logging in', errnote='Unable to log in',
data=json.dumps(auth_data).encode('utf-8'))
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406:
- auth_info = self._parse_json(e.cause.read().encode('utf-8'), None)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 406:
+ auth_info = self._parse_json(e.cause.response.read().encode('utf-8'), None)
else:
raise
diff --git a/yt_dlp/extractor/ign.py b/yt_dlp/extractor/ign.py
index e4db7f9fa..64875f8ce 100644
--- a/yt_dlp/extractor/ign.py
+++ b/yt_dlp/extractor/ign.py
@@ -1,8 +1,9 @@
import re
-import urllib.error
+import urllib.parse
from .common import InfoExtractor
from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
@@ -27,9 +28,9 @@ class IGNBaseIE(InfoExtractor):
try:
return self._call_api(slug)
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 404:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 404:
e.cause.args = e.cause.args or [
- e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+ e.cause.response.url, e.cause.status, e.cause.reason]
raise ExtractorError(
'Content not found: expired?', cause=e.cause,
expected=True)
@@ -226,7 +227,7 @@ class IGNVideoIE(IGNBaseIE):
parsed_url._replace(path=parsed_url.path.rsplit('/', 1)[0] + '/embed'))
webpage, urlh = self._download_webpage_handle(embed_url, video_id)
- new_url = urlh.geturl()
+ new_url = urlh.url
ign_url = compat_parse_qs(
urllib.parse.urlparse(new_url).query).get('url', [None])[-1]
if ign_url:
@@ -323,14 +324,14 @@ class IGNArticleIE(IGNBaseIE):
try:
return self._call_api(slug)
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError):
+ if isinstance(e.cause, HTTPError):
e.cause.args = e.cause.args or [
- e.cause.geturl(), e.cause.getcode(), e.cause.reason]
- if e.cause.code == 404:
+ e.cause.response.url, e.cause.status, e.cause.reason]
+ if e.cause.status == 404:
raise ExtractorError(
'Content not found: expired?', cause=e.cause,
expected=True)
- elif e.cause.code == 503:
+ elif e.cause.status == 503:
self.report_warning(error_to_compat_str(e.cause))
return
raise
diff --git a/yt_dlp/extractor/imggaming.py b/yt_dlp/extractor/imggaming.py
index 8e220fd9f..a40aa2176 100644
--- a/yt_dlp/extractor/imggaming.py
+++ b/yt_dlp/extractor/imggaming.py
@@ -1,7 +1,7 @@
import json
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -52,9 +52,9 @@ class ImgGamingBaseIE(InfoExtractor):
return self._call_api(
stream_path, media_id)['playerUrlCallback']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
raise ExtractorError(
- self._parse_json(e.cause.read().decode(), media_id)['messages'][0],
+ self._parse_json(e.cause.response.read().decode(), media_id)['messages'][0],
expected=True)
raise
diff --git a/yt_dlp/extractor/instagram.py b/yt_dlp/extractor/instagram.py
index 02335138f..bfc4b7b88 100644
--- a/yt_dlp/extractor/instagram.py
+++ b/yt_dlp/extractor/instagram.py
@@ -3,9 +3,9 @@ import itertools
import json
import re
import time
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
decode_base_n,
@@ -442,7 +442,7 @@ class InstagramIE(InstagramBaseIE):
shared_data = self._search_json(
r'window\._sharedData\s*=', webpage, 'shared data', video_id, fatal=False) or {}
- if shared_data and self._LOGIN_URL not in urlh.geturl():
+ if shared_data and self._LOGIN_URL not in urlh.url:
media.update(traverse_obj(
shared_data, ('entry_data', 'PostPage', 0, 'graphql', 'shortcode_media'),
('entry_data', 'PostPage', 0, 'media'), expected_type=dict) or {})
@@ -589,7 +589,7 @@ class InstagramPlaylistBaseIE(InstagramBaseIE):
except ExtractorError as e:
# if it's an error caused by a bad query, and there are
# more GIS templates to try, ignore it and keep trying
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
if gis_tmpl != gis_tmpls[-1]:
continue
raise
diff --git a/yt_dlp/extractor/iprima.py b/yt_dlp/extractor/iprima.py
index e58e9c2ee..6dec1510d 100644
--- a/yt_dlp/extractor/iprima.py
+++ b/yt_dlp/extractor/iprima.py
@@ -81,7 +81,7 @@ class IPrimaIE(InfoExtractor):
note='Logging in')
# a profile may need to be selected first, even when there is only a single one
- if '/profile-select' in login_handle.geturl():
+ if '/profile-select' in login_handle.url:
profile_id = self._search_regex(
r'data-identifier\s*=\s*["\']?(\w+)', profile_select_html, 'profile id')
@@ -89,7 +89,7 @@ class IPrimaIE(InfoExtractor):
f'{self._AUTH_ROOT}/user/profile-select-perform/{profile_id}', None,
query={'continueUrl': '/user/login?redirect_uri=/user/'}, note='Selecting profile')
- code = traverse_obj(login_handle.geturl(), ({parse_qs}, 'code', 0))
+ code = traverse_obj(login_handle.url, ({parse_qs}, 'code', 0))
if not code:
raise ExtractorError('Login failed', expected=True)
diff --git a/yt_dlp/extractor/kakao.py b/yt_dlp/extractor/kakao.py
index 1f0f0a5d5..43055e89d 100644
--- a/yt_dlp/extractor/kakao.py
+++ b/yt_dlp/extractor/kakao.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -101,8 +101,8 @@ class KakaoIE(InfoExtractor):
cdn_api_base, video_id, query=query,
note='Downloading video URL for profile %s' % profile_name)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- resp = self._parse_json(e.cause.read().decode(), video_id)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ resp = self._parse_json(e.cause.response.read().decode(), video_id)
if resp.get('code') == 'GeoBlocked':
self.raise_geo_restricted()
raise
diff --git a/yt_dlp/extractor/kick.py b/yt_dlp/extractor/kick.py
index be1dfd4b1..d12437242 100644
--- a/yt_dlp/extractor/kick.py
+++ b/yt_dlp/extractor/kick.py
@@ -1,7 +1,6 @@
from .common import InfoExtractor
-
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
UserNotLive,
float_or_none,
merge_dicts,
diff --git a/yt_dlp/extractor/kuwo.py b/yt_dlp/extractor/kuwo.py
index cfec1c50f..e8a061a10 100644
--- a/yt_dlp/extractor/kuwo.py
+++ b/yt_dlp/extractor/kuwo.py
@@ -91,7 +91,7 @@ class KuwoIE(KuwoBaseIE):
webpage, urlh = self._download_webpage_handle(
url, song_id, note='Download song detail info',
errnote='Unable to get song detail info')
- if song_id not in urlh.geturl() or '对不起,该歌曲由于版权问题已被下线,将返回网站首页' in webpage:
+ if song_id not in urlh.url or '对不起,该歌曲由于版权问题已被下线,将返回网站首页' in webpage:
raise ExtractorError('this song has been offline because of copyright issues', expected=True)
song_name = self._html_search_regex(
diff --git a/yt_dlp/extractor/la7.py b/yt_dlp/extractor/la7.py
index 36bfaf5c3..a3cd12b00 100644
--- a/yt_dlp/extractor/la7.py
+++ b/yt_dlp/extractor/la7.py
@@ -1,13 +1,8 @@
import re
from .common import InfoExtractor
-from ..utils import (
- float_or_none,
- HEADRequest,
- int_or_none,
- parse_duration,
- unified_strdate,
-)
+from ..networking import HEADRequest
+from ..utils import float_or_none, int_or_none, parse_duration, unified_strdate
class LA7IE(InfoExtractor):
diff --git a/yt_dlp/extractor/lbry.py b/yt_dlp/extractor/lbry.py
index 23d3daf13..6af64f0df 100644
--- a/yt_dlp/extractor/lbry.py
+++ b/yt_dlp/extractor/lbry.py
@@ -3,9 +3,9 @@ import json
import urllib.parse
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
OnDemandPagedList,
UnsupportedError,
determine_ext,
@@ -266,7 +266,7 @@ class LBRYIE(LBRYBaseIE):
# HEAD request returns redirect response to m3u8 URL if available
final_url = self._request_webpage(
HEADRequest(streaming_url), display_id, headers=headers,
- note='Downloading streaming redirect url info').geturl()
+ note='Downloading streaming redirect url info').url
elif result.get('value_type') == 'stream':
claim_id, is_live = result['signing_channel']['claim_id'], True
diff --git a/yt_dlp/extractor/lecturio.py b/yt_dlp/extractor/lecturio.py
index 973764c63..bb059d3a2 100644
--- a/yt_dlp/extractor/lecturio.py
+++ b/yt_dlp/extractor/lecturio.py
@@ -25,7 +25,7 @@ class LecturioBaseIE(InfoExtractor):
self._LOGIN_URL, None, 'Downloading login popup')
def is_logged(url_handle):
- return self._LOGIN_URL not in url_handle.geturl()
+ return self._LOGIN_URL not in url_handle.url
# Already logged in
if is_logged(urlh):
diff --git a/yt_dlp/extractor/lego.py b/yt_dlp/extractor/lego.py
index 811b44758..46fc7a9b6 100644
--- a/yt_dlp/extractor/lego.py
+++ b/yt_dlp/extractor/lego.py
@@ -1,7 +1,7 @@
import uuid
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -75,7 +75,7 @@ class LEGOIE(InfoExtractor):
'videoId': '%s_%s' % (uuid.UUID(video_id), locale),
}, headers=self.geo_verification_headers())
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 451:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 451:
self.raise_geo_restricted(countries=countries)
raise
diff --git a/yt_dlp/extractor/limelight.py b/yt_dlp/extractor/limelight.py
index e11ec43d6..4e50f106f 100644
--- a/yt_dlp/extractor/limelight.py
+++ b/yt_dlp/extractor/limelight.py
@@ -1,7 +1,7 @@
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
float_or_none,
@@ -69,8 +69,8 @@ class LimelightBaseIE(InfoExtractor):
item_id, 'Downloading PlaylistService %s JSON' % method,
fatal=fatal, headers=headers)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- error = self._parse_json(e.cause.read().decode(), item_id)['detail']['contentAccessPermission']
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ error = self._parse_json(e.cause.response.read().decode(), item_id)['detail']['contentAccessPermission']
if error == 'CountryDisabled':
self.raise_geo_restricted()
raise ExtractorError(error, expected=True)
diff --git a/yt_dlp/extractor/linuxacademy.py b/yt_dlp/extractor/linuxacademy.py
index 7bb64e17c..0b1644293 100644
--- a/yt_dlp/extractor/linuxacademy.py
+++ b/yt_dlp/extractor/linuxacademy.py
@@ -2,11 +2,8 @@ import json
import random
from .common import InfoExtractor
-from ..compat import (
- compat_b64decode,
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_b64decode, compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
ExtractorError,
@@ -107,7 +104,7 @@ class LinuxAcademyIE(InfoExtractor):
'sso': 'true',
})
- login_state_url = urlh.geturl()
+ login_state_url = urlh.url
try:
login_page = self._download_webpage(
@@ -119,8 +116,8 @@ class LinuxAcademyIE(InfoExtractor):
'Referer': login_state_url,
})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read(), None)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read(), None)
message = error.get('description') or error['code']
raise ExtractorError(
'%s said: %s' % (self.IE_NAME, message), expected=True)
@@ -137,7 +134,7 @@ class LinuxAcademyIE(InfoExtractor):
})
access_token = self._search_regex(
- r'access_token=([^=&]+)', urlh.geturl(),
+ r'access_token=([^=&]+)', urlh.url,
'access token', default=None)
if not access_token:
access_token = self._parse_json(
diff --git a/yt_dlp/extractor/mediasite.py b/yt_dlp/extractor/mediasite.py
index fe549c49f..7ea78ab69 100644
--- a/yt_dlp/extractor/mediasite.py
+++ b/yt_dlp/extractor/mediasite.py
@@ -171,7 +171,7 @@ class MediasiteIE(InfoExtractor):
query = mobj.group('query')
webpage, urlh = self._download_webpage_handle(url, resource_id) # XXX: add UrlReferrer?
- redirect_url = urlh.geturl()
+ redirect_url = urlh.url
# XXX: might have also extracted UrlReferrer and QueryString from the html
service_path = compat_urlparse.urljoin(redirect_url, self._html_search_regex(
diff --git a/yt_dlp/extractor/megatvcom.py b/yt_dlp/extractor/megatvcom.py
index 2f3f11f51..93c7e7dc0 100644
--- a/yt_dlp/extractor/megatvcom.py
+++ b/yt_dlp/extractor/megatvcom.py
@@ -1,14 +1,14 @@
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
+ ExtractorError,
clean_html,
determine_ext,
- ExtractorError,
extract_attributes,
get_element_by_class,
get_element_html_by_id,
- HEADRequest,
parse_qs,
unescapeHTML,
unified_timestamp,
@@ -160,5 +160,5 @@ class MegaTVComEmbedIE(MegaTVComBaseIE):
canonical_url = self._request_webpage(
HEADRequest(canonical_url), video_id,
note='Resolve canonical URL',
- errnote='Could not resolve canonical URL').geturl()
+ errnote='Could not resolve canonical URL').url
return self.url_result(canonical_url, MegaTVComIE.ie_key(), video_id)
diff --git a/yt_dlp/extractor/mgtv.py b/yt_dlp/extractor/mgtv.py
index 06edcb396..31ccf004e 100644
--- a/yt_dlp/extractor/mgtv.py
+++ b/yt_dlp/extractor/mgtv.py
@@ -1,9 +1,9 @@
import base64
import time
-import urllib.error
import uuid
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -86,8 +86,8 @@ class MGTVIE(InfoExtractor):
'type': 'pch5'
}, headers=self.geo_verification_headers())['data']
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read().decode(), None)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read().decode(), None)
if error.get('code') == 40005:
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
raise ExtractorError(error['msg'], expected=True)
diff --git a/yt_dlp/extractor/minds.py b/yt_dlp/extractor/minds.py
index 2fb17920c..27a6e3805 100644
--- a/yt_dlp/extractor/minds.py
+++ b/yt_dlp/extractor/minds.py
@@ -106,7 +106,7 @@ class MindsIE(MindsBaseIE):
if poster:
urlh = self._request_webpage(poster, video_id, fatal=False)
if urlh:
- thumbnail = urlh.geturl()
+ thumbnail = urlh.url
return {
'id': video_id,
diff --git a/yt_dlp/extractor/miomio.py b/yt_dlp/extractor/miomio.py
index a0a041ea5..8df8cba19 100644
--- a/yt_dlp/extractor/miomio.py
+++ b/yt_dlp/extractor/miomio.py
@@ -2,12 +2,8 @@ import random
from .common import InfoExtractor
from ..compat import compat_urlparse
-from ..utils import (
- xpath_text,
- int_or_none,
- ExtractorError,
- sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError, int_or_none, xpath_text
class MioMioIE(InfoExtractor):
@@ -61,7 +57,7 @@ class MioMioIE(InfoExtractor):
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
video_id)
- vid_config_request = sanitized_Request(
+ vid_config_request = Request(
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
headers=http_headers)
diff --git a/yt_dlp/extractor/mtv.py b/yt_dlp/extractor/mtv.py
index d91be6270..0d700b9a8 100644
--- a/yt_dlp/extractor/mtv.py
+++ b/yt_dlp/extractor/mtv.py
@@ -2,16 +2,15 @@ import re
from .common import InfoExtractor
from ..compat import compat_str
+from ..networking import HEADRequest, Request
from ..utils import (
ExtractorError,
+ RegexNotFoundError,
find_xpath_attr,
fix_xml_ampersands,
float_or_none,
- HEADRequest,
int_or_none,
join_nonempty,
- RegexNotFoundError,
- sanitized_Request,
strip_or_none,
timeconvert,
try_get,
@@ -51,15 +50,15 @@ class MTVServicesInfoExtractor(InfoExtractor):
def _extract_mobile_video_formats(self, mtvn_id):
webpage_url = self._MOBILE_TEMPLATE % mtvn_id
- req = sanitized_Request(webpage_url)
+ req = Request(webpage_url)
# Otherwise we get a webpage that would execute some javascript
- req.add_header('User-Agent', 'curl/7')
+ req.headers['User-Agent'] = 'curl/7'
webpage = self._download_webpage(req, mtvn_id,
'Downloading mobile page')
metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
req = HEADRequest(metrics_url)
response = self._request_webpage(req, mtvn_id, 'Resolving url')
- url = response.geturl()
+ url = response.url
# Transform the url to get the best quality:
url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
return [{'url': url, 'ext': 'mp4'}]
diff --git a/yt_dlp/extractor/nbc.py b/yt_dlp/extractor/nbc.py
index ddc89a7c2..299b05174 100644
--- a/yt_dlp/extractor/nbc.py
+++ b/yt_dlp/extractor/nbc.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
from .theplatform import ThePlatformIE, default_ns
from .adobepass import AdobePassIE
from ..compat import compat_urllib_parse_unquote
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
RegexNotFoundError,
UserNotLive,
clean_html,
diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py
index 7a5a02dfa..4f3e691b7 100644
--- a/yt_dlp/extractor/nebula.py
+++ b/yt_dlp/extractor/nebula.py
@@ -1,8 +1,8 @@
import itertools
import json
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import ExtractorError, make_archive_id, parse_iso8601, remove_start
_BASE_URL_RE = r'https?://(?:www\.|beta\.)?(?:watchnebula\.com|nebula\.app|nebula\.tv)'
@@ -48,7 +48,7 @@ class NebulaBaseIE(InfoExtractor):
return inner_call()
except ExtractorError as exc:
# if 401 or 403, attempt credential re-auth and retry
- if exc.cause and isinstance(exc.cause, urllib.error.HTTPError) and exc.cause.code in (401, 403):
+ if exc.cause and isinstance(exc.cause, HTTPError) and exc.cause.status in (401, 403):
self.to_screen(f'Reauthenticating to Nebula and retrying, because last {auth_type} call resulted in error {exc.cause.code}')
self._perform_login()
return inner_call()
diff --git a/yt_dlp/extractor/neteasemusic.py b/yt_dlp/extractor/neteasemusic.py
index 595709899..5b7307bc8 100644
--- a/yt_dlp/extractor/neteasemusic.py
+++ b/yt_dlp/extractor/neteasemusic.py
@@ -11,6 +11,7 @@ from random import randint
from .common import InfoExtractor
from ..aes import aes_ecb_encrypt, pkcs7_padding
from ..compat import compat_urllib_parse_urlencode
+from ..networking import Request
from ..utils import (
ExtractorError,
bytes_to_intlist,
@@ -18,7 +19,6 @@ from ..utils import (
float_or_none,
int_or_none,
intlist_to_bytes,
- sanitized_Request,
try_get,
)
@@ -146,8 +146,8 @@ class NetEaseMusicBaseIE(InfoExtractor):
return int(round(ms / 1000.0))
def query_api(self, endpoint, video_id, note):
- req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
- req.add_header('Referer', self._API_BASE)
+ req = Request('%s%s' % (self._API_BASE, endpoint))
+ req.headers['Referer'] = self._API_BASE
return self._download_json(req, video_id, note)
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index 89e8e6093..fa2d709d2 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -8,10 +8,8 @@ import time
from urllib.parse import urlparse
from .common import InfoExtractor, SearchInfoExtractor
-from ..compat import (
- compat_HTTPError,
-)
from ..dependencies import websockets
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
OnDemandPagedList,
@@ -396,7 +394,7 @@ class NiconicoIE(InfoExtractor):
webpage, handle = self._download_webpage_handle(
'https://www.nicovideo.jp/watch/' + video_id, video_id)
if video_id.startswith('so'):
- video_id = self._match_id(handle.geturl())
+ video_id = self._match_id(handle.url)
api_data = self._parse_json(self._html_search_regex(
'data-api-data="([^"]+)"', webpage,
@@ -407,9 +405,9 @@ class NiconicoIE(InfoExtractor):
'https://www.nicovideo.jp/api/watch/v3/%s?_frontendId=6&_frontendVersion=0&actionTrackId=AAAAAAAAAA_%d' % (video_id, round(time.time() * 1000)), video_id,
note='Downloading API JSON', errnote='Unable to fetch data')['data']
except ExtractorError:
- if not isinstance(e.cause, compat_HTTPError):
+ if not isinstance(e.cause, HTTPError):
raise
- webpage = e.cause.read().decode('utf-8', 'replace')
+ webpage = e.cause.response.read().decode('utf-8', 'replace')
error_msg = self._html_search_regex(
r'(?s)<section\s+class="(?:(?:ErrorMessage|WatchExceptionPage-message)\s*)+">(.+?)</section>',
webpage, 'error reason', default=None)
@@ -742,7 +740,7 @@ class NiconicoHistoryIE(NiconicoPlaylistBaseIE):
try:
mylist = self._call_api(list_id, 'list', {'pageSize': 1})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
self.raise_login_required('You have to be logged in to get your history')
raise
return self.playlist_result(self._entries(list_id), list_id, **self._parse_owner(mylist))
@@ -951,8 +949,8 @@ class NiconicoLiveIE(InfoExtractor):
'frontend_id': traverse_obj(embedded_data, ('site', 'frontendId')) or '9',
})
- hostname = remove_start(urlparse(urlh.geturl()).hostname, 'sp.')
- cookies = try_get(urlh.geturl(), self._downloader._calc_cookies)
+ hostname = remove_start(urlparse(urlh.url).hostname, 'sp.')
+ cookies = try_get(urlh.url, self._downloader._calc_cookies)
latency = try_get(self._configuration_arg('latency'), lambda x: x[0])
if latency not in self._KNOWN_LATENCY:
latency = 'high'
diff --git a/yt_dlp/extractor/njpwworld.py b/yt_dlp/extractor/njpwworld.py
index 7b8a526f0..607838133 100644
--- a/yt_dlp/extractor/njpwworld.py
+++ b/yt_dlp/extractor/njpwworld.py
@@ -51,7 +51,7 @@ class NJPWWorldIE(InfoExtractor):
data=urlencode_postdata({'login_id': username, 'pw': password}),
headers={'Referer': 'https://front.njpwworld.com/auth'})
# /auth/login will return 302 for successful logins
- if urlh.geturl() == self._LOGIN_URL:
+ if urlh.url == self._LOGIN_URL:
self.report_warning('unable to login')
return False
diff --git a/yt_dlp/extractor/nosvideo.py b/yt_dlp/extractor/nosvideo.py
index b6d3ea40c..7e9688c0b 100644
--- a/yt_dlp/extractor/nosvideo.py
+++ b/yt_dlp/extractor/nosvideo.py
@@ -1,9 +1,9 @@
import re
from .common import InfoExtractor
+from ..networking import Request
from ..utils import (
ExtractorError,
- sanitized_Request,
urlencode_postdata,
xpath_text,
xpath_with_ns,
@@ -36,8 +36,8 @@ class NosVideoIE(InfoExtractor):
'op': 'download1',
'method_free': 'Continue to Video',
}
- req = sanitized_Request(url, urlencode_postdata(fields))
- req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ req = Request(url, urlencode_postdata(fields))
+ req.headers['Content-type'] = 'application/x-www-form-urlencoded'
webpage = self._download_webpage(req, video_id,
'Downloading download page')
if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
diff --git a/yt_dlp/extractor/nowness.py b/yt_dlp/extractor/nowness.py
index fc9043bce..a3c29f62c 100644
--- a/yt_dlp/extractor/nowness.py
+++ b/yt_dlp/extractor/nowness.py
@@ -4,10 +4,8 @@ from .brightcove import (
)
from .common import InfoExtractor
from ..compat import compat_str
-from ..utils import (
- ExtractorError,
- sanitized_Request,
-)
+from ..networking import Request
+from ..utils import ExtractorError
class NownessBaseIE(InfoExtractor):
@@ -40,7 +38,7 @@ class NownessBaseIE(InfoExtractor):
def _api_request(self, url, request_path):
display_id = self._match_id(url)
- request = sanitized_Request(
+ request = Request(
'http://api.nowness.com/api/' + request_path % display_id,
headers={
'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
diff --git a/yt_dlp/extractor/nrk.py b/yt_dlp/extractor/nrk.py
index 88d08e5e3..384865acc 100644
--- a/yt_dlp/extractor/nrk.py
+++ b/yt_dlp/extractor/nrk.py
@@ -3,7 +3,8 @@ import random
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
@@ -148,7 +149,7 @@ class NRKIE(NRKBaseIE):
try:
return self._call_api(f'playback/{item}/program/{video_id}', video_id, item, query=query)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
return self._call_api(f'playback/{item}/{video_id}', video_id, item, query=query)
raise
diff --git a/yt_dlp/extractor/odkmedia.py b/yt_dlp/extractor/odkmedia.py
index 2960860d6..b852160b9 100644
--- a/yt_dlp/extractor/odkmedia.py
+++ b/yt_dlp/extractor/odkmedia.py
@@ -1,7 +1,7 @@
import json
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
GeoRestrictedError,
@@ -74,8 +74,8 @@ class OnDemandChinaEpisodeIE(InfoExtractor):
f'https://odkmedia.io/odc/api/v2/playback/{video_info["id"]}/', display_id,
headers={'Authorization': '', 'service-name': 'odc'})
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError):
- error_data = self._parse_json(e.cause.read(), display_id)['detail']
+ if isinstance(e.cause, HTTPError):
+ error_data = self._parse_json(e.cause.response.read(), display_id)['detail']
raise GeoRestrictedError(error_data)
formats, subtitles = [], {}
diff --git a/yt_dlp/extractor/odnoklassniki.py b/yt_dlp/extractor/odnoklassniki.py
index e63714e84..1be45d8ad 100644
--- a/yt_dlp/extractor/odnoklassniki.py
+++ b/yt_dlp/extractor/odnoklassniki.py
@@ -7,9 +7,9 @@ from ..compat import (
compat_urllib_parse_unquote,
compat_urllib_parse_urlparse,
)
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
float_or_none,
int_or_none,
qualities,
@@ -448,7 +448,7 @@ class OdnoklassnikiIE(InfoExtractor):
json_data = self._parse_json(unescapeHTML(json_data), video_id) or {}
redirect_url = self._request_webpage(HEADRequest(
- json_data['videoSrc']), video_id, 'Requesting download URL').geturl()
+ json_data['videoSrc']), video_id, 'Requesting download URL').url
self._clear_cookies(redirect_url)
return {
diff --git a/yt_dlp/extractor/orf.py b/yt_dlp/extractor/orf.py
index e9d23a4d1..cc3c003fa 100644
--- a/yt_dlp/extractor/orf.py
+++ b/yt_dlp/extractor/orf.py
@@ -2,11 +2,11 @@ import functools
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
clean_html,
determine_ext,
float_or_none,
- HEADRequest,
InAdvancePagedList,
int_or_none,
join_nonempty,
diff --git a/yt_dlp/extractor/owncloud.py b/yt_dlp/extractor/owncloud.py
index e1d5682f8..79fd830bb 100644
--- a/yt_dlp/extractor/owncloud.py
+++ b/yt_dlp/extractor/owncloud.py
@@ -44,7 +44,7 @@ class OwnCloudIE(InfoExtractor):
webpage, urlh = self._download_webpage_handle(url, video_id)
if re.search(r'<label[^>]+for="password"', webpage):
- webpage = self._verify_video_password(webpage, urlh.geturl(), video_id)
+ webpage = self._verify_video_password(webpage, urlh.url, video_id)
hidden_inputs = self._hidden_inputs(webpage)
title = hidden_inputs.get('filename')
diff --git a/yt_dlp/extractor/packtpub.py b/yt_dlp/extractor/packtpub.py
index 51778d8a2..56203306f 100644
--- a/yt_dlp/extractor/packtpub.py
+++ b/yt_dlp/extractor/packtpub.py
@@ -1,10 +1,7 @@
import json
from .common import InfoExtractor
-from ..compat import (
- # compat_str,
- compat_HTTPError,
-)
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
ExtractorError,
@@ -54,8 +51,8 @@ class PacktPubIE(PacktPubBaseIE):
'password': password,
}).encode())['data']['access']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 404):
- message = self._parse_json(e.cause.read().decode(), None)['message']
+ if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 404):
+ message = self._parse_json(e.cause.response.read().decode(), None)['message']
raise ExtractorError(message, expected=True)
raise
@@ -70,7 +67,7 @@ class PacktPubIE(PacktPubBaseIE):
'https://services.packtpub.com/products-v1/products/%s/%s/%s' % (course_id, chapter_id, video_id), video_id,
'Downloading JSON video', headers=headers)['data']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
self.raise_login_required('This video is locked')
raise
diff --git a/yt_dlp/extractor/patreon.py b/yt_dlp/extractor/patreon.py
index e93e37eb9..447087436 100644
--- a/yt_dlp/extractor/patreon.py
+++ b/yt_dlp/extractor/patreon.py
@@ -1,10 +1,10 @@
import itertools
-from urllib.error import HTTPError
from .common import InfoExtractor
from .vimeo import VimeoIE
from ..compat import compat_urllib_parse_unquote
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
determine_ext,
@@ -37,9 +37,9 @@ class PatreonBaseIE(InfoExtractor):
item_id, note='Downloading API JSON' if not note else note,
query=query, fatal=fatal, headers=headers)
except ExtractorError as e:
- if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.headers.get('Content-Type')) != 'json':
+ if not isinstance(e.cause, HTTPError) or mimetype2ext(e.cause.response.headers.get('Content-Type')) != 'json':
raise
- err_json = self._parse_json(self._webpage_read_content(e.cause, None, item_id), item_id, fatal=False)
+ err_json = self._parse_json(self._webpage_read_content(e.cause.response, None, item_id), item_id, fatal=False)
err_message = traverse_obj(err_json, ('errors', ..., 'detail'), get_all=False)
if err_message:
raise ExtractorError(f'Patreon said: {err_message}', expected=True)
diff --git a/yt_dlp/extractor/peloton.py b/yt_dlp/extractor/peloton.py
index 4835822cf..786429988 100644
--- a/yt_dlp/extractor/peloton.py
+++ b/yt_dlp/extractor/peloton.py
@@ -3,7 +3,7 @@ import re
import urllib.parse
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
float_or_none,
@@ -83,8 +83,8 @@ class PelotonIE(InfoExtractor):
}).encode(),
headers={'Content-Type': 'application/json', 'User-Agent': 'web'})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- json_string = self._webpage_read_content(e.cause, None, video_id)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ json_string = self._webpage_read_content(e.cause.response, None, video_id)
res = self._parse_json(json_string, video_id)
raise ExtractorError(res['message'], expected=res['message'] == 'Login failed')
else:
@@ -96,8 +96,8 @@ class PelotonIE(InfoExtractor):
'https://api.onepeloton.com/api/subscription/stream', video_id, note='Downloading token',
data=json.dumps({}).encode(), headers={'Content-Type': 'application/json'})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- json_string = self._webpage_read_content(e.cause, None, video_id)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ json_string = self._webpage_read_content(e.cause.response, None, video_id)
res = self._parse_json(json_string, video_id)
raise ExtractorError(res['message'], expected=res['message'] == 'Stream limit reached')
else:
@@ -109,7 +109,7 @@ class PelotonIE(InfoExtractor):
try:
self._start_session(video_id)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
self._login(video_id)
self._start_session(video_id)
else:
diff --git a/yt_dlp/extractor/piapro.py b/yt_dlp/extractor/piapro.py
index eb5923d11..5f39e0639 100644
--- a/yt_dlp/extractor/piapro.py
+++ b/yt_dlp/extractor/piapro.py
@@ -69,7 +69,7 @@ class PiaproIE(InfoExtractor):
if urlh is False:
login_ok = False
else:
- parts = compat_urlparse.urlparse(urlh.geturl())
+ parts = compat_urlparse.urlparse(urlh.url)
if parts.path != '/':
login_ok = False
if not login_ok:
diff --git a/yt_dlp/extractor/pladform.py b/yt_dlp/extractor/pladform.py
index dcf18e1f3..00500686f 100644
--- a/yt_dlp/extractor/pladform.py
+++ b/yt_dlp/extractor/pladform.py
@@ -78,7 +78,7 @@ class PladformIE(InfoExtractor):
expected=True)
if not video:
- targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').geturl()
+ targetUrl = self._request_webpage(url, video_id, note='Resolving final URL').url
if targetUrl == url:
raise ExtractorError('Can\'t parse page')
return self.url_result(targetUrl)
diff --git a/yt_dlp/extractor/platzi.py b/yt_dlp/extractor/platzi.py
index b8a441494..166b98c4a 100644
--- a/yt_dlp/extractor/platzi.py
+++ b/yt_dlp/extractor/platzi.py
@@ -36,7 +36,7 @@ class PlatziBaseIE(InfoExtractor):
headers={'Referer': self._LOGIN_URL})
# login succeeded
- if 'platzi.com/login' not in urlh.geturl():
+ if 'platzi.com/login' not in urlh.url:
return
login_error = self._webpage_read_content(
diff --git a/yt_dlp/extractor/playplustv.py b/yt_dlp/extractor/playplustv.py
index 316f220f7..a4439c8bc 100644
--- a/yt_dlp/extractor/playplustv.py
+++ b/yt_dlp/extractor/playplustv.py
@@ -1,13 +1,9 @@
import json
from .common import InfoExtractor
-from ..compat import compat_HTTPError
-from ..utils import (
- clean_html,
- ExtractorError,
- int_or_none,
- PUTRequest,
-)
+from ..networking import PUTRequest
+from ..networking.exceptions import HTTPError
+from ..utils import ExtractorError, clean_html, int_or_none
class PlayPlusTVIE(InfoExtractor):
@@ -47,9 +43,9 @@ class PlayPlusTVIE(InfoExtractor):
try:
self._token = self._download_json(req, None)['token']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
raise ExtractorError(self._parse_json(
- e.cause.read(), None)['errorMessage'], expected=True)
+ e.cause.response.read(), None)['errorMessage'], expected=True)
raise
self._profile = self._call_api('Profiles')['list'][0]['_id']
diff --git a/yt_dlp/extractor/pornhub.py b/yt_dlp/extractor/pornhub.py
index 2f5a572a5..f08414030 100644
--- a/yt_dlp/extractor/pornhub.py
+++ b/yt_dlp/extractor/pornhub.py
@@ -3,11 +3,12 @@ import itertools
import math
import operator
import re
-import urllib.request
from .common import InfoExtractor
from .openload import PhantomJSwrapper
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking import Request
+from ..networking.exceptions import HTTPError
from ..utils import (
NO_DEFAULT,
ExtractorError,
@@ -46,8 +47,8 @@ class PornHubBaseIE(InfoExtractor):
r'document\.cookie\s*=\s*["\']RNKEY=',
r'document\.location\.reload\(true\)')):
url_or_request = args[0]
- url = (url_or_request.get_full_url()
- if isinstance(url_or_request, urllib.request.Request)
+ url = (url_or_request.url
+ if isinstance(url_or_request, Request)
else url_or_request)
phantom = PhantomJSwrapper(self, required_version='2.0')
phantom.get(url, html=webpage)
@@ -602,7 +603,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
base_url, item_id, note, query={'page': num})
def is_404(e):
- return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
+ return isinstance(e.cause, HTTPError) and e.cause.status == 404
base_url = url
has_page = page is not None
diff --git a/yt_dlp/extractor/puhutv.py b/yt_dlp/extractor/puhutv.py
index 482e5705f..4b8e5e90d 100644
--- a/yt_dlp/extractor/puhutv.py
+++ b/yt_dlp/extractor/puhutv.py
@@ -1,8 +1,6 @@
from .common import InfoExtractor
-from ..compat import (
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -72,7 +70,7 @@ class PuhuTVIE(InfoExtractor):
display_id, 'Downloading video JSON',
headers=self.geo_verification_headers())
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
self.raise_geo_restricted()
raise
diff --git a/yt_dlp/extractor/radiko.py b/yt_dlp/extractor/radiko.py
index 7fdf78283..cef68eba0 100644
--- a/yt_dlp/extractor/radiko.py
+++ b/yt_dlp/extractor/radiko.py
@@ -41,7 +41,7 @@ class RadikoBaseIE(InfoExtractor):
'x-radiko-device': 'pc',
'x-radiko-user': 'dummy_user',
})
- auth1_header = auth1_handle.info()
+ auth1_header = auth1_handle.headers
auth_token = auth1_header['X-Radiko-AuthToken']
kl = int(auth1_header['X-Radiko-KeyLength'])
diff --git a/yt_dlp/extractor/radiocanada.py b/yt_dlp/extractor/radiocanada.py
index 72c21d502..1a5a6355a 100644
--- a/yt_dlp/extractor/radiocanada.py
+++ b/yt_dlp/extractor/radiocanada.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -74,8 +74,8 @@ class RadioCanadaIE(InfoExtractor):
return self._download_json(
'https://services.radio-canada.ca/media/' + path, video_id, query=query)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 422):
- data = self._parse_json(e.cause.read().decode(), None)
+ if isinstance(e.cause, HTTPError) and e.cause.status in (401, 422):
+ data = self._parse_json(e.cause.response.read().decode(), None)
error = data.get('error_description') or data['errorMessage']['text']
raise ExtractorError(error, expected=True)
raise
diff --git a/yt_dlp/extractor/rcs.py b/yt_dlp/extractor/rcs.py
index 2440858ca..028d3d90b 100644
--- a/yt_dlp/extractor/rcs.py
+++ b/yt_dlp/extractor/rcs.py
@@ -1,9 +1,9 @@
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
base_url,
clean_html,
extract_attributes,
diff --git a/yt_dlp/extractor/rcti.py b/yt_dlp/extractor/rcti.py
index 27b4ad7bb..79d9c8e31 100644
--- a/yt_dlp/extractor/rcti.py
+++ b/yt_dlp/extractor/rcti.py
@@ -3,7 +3,7 @@ import random
import time
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
dict_get,
ExtractorError,
@@ -186,7 +186,7 @@ class RCTIPlusIE(RCTIPlusBaseIE):
try:
formats = self._extract_m3u8_formats(video_url, display_id, 'mp4', headers={'Referer': 'https://www.rctiplus.com/'})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
self.raise_geo_restricted(countries=['ID'], metadata_available=True)
else:
raise e
diff --git a/yt_dlp/extractor/recurbate.py b/yt_dlp/extractor/recurbate.py
index 5534cf3c3..d7294cb14 100644
--- a/yt_dlp/extractor/recurbate.py
+++ b/yt_dlp/extractor/recurbate.py
@@ -1,6 +1,5 @@
-import urllib.error
-
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import ExtractorError, merge_dicts
@@ -25,7 +24,7 @@ class RecurbateIE(InfoExtractor):
try:
webpage = self._download_webpage(url, video_id)
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
self.raise_login_required(msg=SUBSCRIPTION_MISSING_MESSAGE, method='cookies')
raise
token = self._html_search_regex(r'data-token="([^"]+)"', webpage, 'token')
diff --git a/yt_dlp/extractor/redbulltv.py b/yt_dlp/extractor/redbulltv.py
index a01bc8434..d1de2490f 100644
--- a/yt_dlp/extractor/redbulltv.py
+++ b/yt_dlp/extractor/redbulltv.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
float_or_none,
ExtractorError,
@@ -68,9 +68,9 @@ class RedBullTVIE(InfoExtractor):
headers={'Authorization': token}
)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 404:
error_message = self._parse_json(
- e.cause.read().decode(), video_id)['error']
+ e.cause.response.read().decode(), video_id)['error']
raise ExtractorError('%s said: %s' % (
self.IE_NAME, error_message), expected=True)
raise
diff --git a/yt_dlp/extractor/redgifs.py b/yt_dlp/extractor/redgifs.py
index 098fb8185..f9453202b 100644
--- a/yt_dlp/extractor/redgifs.py
+++ b/yt_dlp/extractor/redgifs.py
@@ -1,8 +1,8 @@
import functools
-import urllib
from .common import InfoExtractor
from ..compat import compat_parse_qs
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -82,7 +82,7 @@ class RedGifsBaseInfoExtractor(InfoExtractor):
f'https://api.redgifs.com/v2/{ep}', video_id, headers=headers, *args, **kwargs)
break
except ExtractorError as e:
- if first_attempt and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+ if first_attempt and isinstance(e.cause, HTTPError) and e.cause.status == 401:
del self._API_HEADERS['authorization'] # refresh the token
continue
raise
diff --git a/yt_dlp/extractor/regiotv.py b/yt_dlp/extractor/regiotv.py
index 6114841fb..edb6ae5bc 100644
--- a/yt_dlp/extractor/regiotv.py
+++ b/yt_dlp/extractor/regiotv.py
@@ -1,10 +1,6 @@
from .common import InfoExtractor
-
-from ..utils import (
- sanitized_Request,
- xpath_text,
- xpath_with_ns,
-)
+from ..networking import Request
+from ..utils import xpath_text, xpath_with_ns
class RegioTVIE(InfoExtractor):
@@ -33,7 +29,7 @@ class RegioTVIE(InfoExtractor):
SOAP_TEMPLATE = '<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><{0} xmlns="http://v.telvi.de/"><key xsi:type="xsd:string">{1}</key></{0}></soap:Body></soap:Envelope>'
- request = sanitized_Request(
+ request = Request(
'http://v.telvi.de/',
SOAP_TEMPLATE.format('GetHTML5VideoData', key).encode('utf-8'))
video_data = self._download_xml(request, video_id, 'Downloading video XML')
diff --git a/yt_dlp/extractor/rokfin.py b/yt_dlp/extractor/rokfin.py
index 0e40eb32a..4a4d40bef 100644
--- a/yt_dlp/extractor/rokfin.py
+++ b/yt_dlp/extractor/rokfin.py
@@ -245,7 +245,7 @@ class RokfinIE(InfoExtractor):
f'{self._AUTH_BASE}/token', None,
note='getting access credentials', errnote='error getting access credentials',
data=urlencode_postdata({
- 'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.geturl()).fragment).get('code')[0],
+ 'code': urllib.parse.parse_qs(urllib.parse.urldefrag(urlh.url).fragment).get('code')[0],
'client_id': 'web',
'grant_type': 'authorization_code',
'redirect_uri': 'https://rokfin.com/silent-check-sso.html'
@@ -269,7 +269,7 @@ class RokfinIE(InfoExtractor):
json_string, urlh = self._download_webpage_handle(
url_or_request, video_id, headers=headers, query=query, expected_status=401)
- if not auth_token or urlh.code != 401 or refresh_token is None:
+ if not auth_token or urlh.status != 401 or refresh_token is None:
return self._parse_json(json_string, video_id)
self._access_mgmt_tokens = self._download_json(
diff --git a/yt_dlp/extractor/roosterteeth.py b/yt_dlp/extractor/roosterteeth.py
index 776fbfbc0..94e673b13 100644
--- a/yt_dlp/extractor/roosterteeth.py
+++ b/yt_dlp/extractor/roosterteeth.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -35,8 +35,8 @@ class RoosterTeethBaseIE(InfoExtractor):
}))
except ExtractorError as e:
msg = 'Unable to login'
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- resp = self._parse_json(e.cause.read().decode(), None, fatal=False)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ resp = self._parse_json(e.cause.response.read().decode(), None, fatal=False)
if resp:
error = resp.get('extra_info') or resp.get('error_description') or resp.get('error')
if error:
@@ -138,8 +138,8 @@ class RoosterTeethIE(RoosterTeethBaseIE):
m3u8_url = video_data['attributes']['url']
# XXX: additional URL at video_data['links']['download']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- if self._parse_json(e.cause.read().decode(), display_id).get('access') is False:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ if self._parse_json(e.cause.response.read().decode(), display_id).get('access') is False:
self.raise_login_required(
'%s is only available for FIRST members' % display_id)
raise
diff --git a/yt_dlp/extractor/rozhlas.py b/yt_dlp/extractor/rozhlas.py
index 5f83d42e8..63134322d 100644
--- a/yt_dlp/extractor/rozhlas.py
+++ b/yt_dlp/extractor/rozhlas.py
@@ -1,7 +1,7 @@
import itertools
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
extract_attributes,
@@ -81,7 +81,7 @@ class RozhlasBaseIE(InfoExtractor):
'vcodec': 'none',
})
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 429:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 429:
retry.error = e.cause
else:
self.report_warning(e.msg)
diff --git a/yt_dlp/extractor/rte.py b/yt_dlp/extractor/rte.py
index aedaa5b55..7ba80d4ba 100644
--- a/yt_dlp/extractor/rte.py
+++ b/yt_dlp/extractor/rte.py
@@ -1,7 +1,7 @@
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
float_or_none,
parse_iso8601,
@@ -31,8 +31,8 @@ class RteBaseIE(InfoExtractor):
except ExtractorError as ee:
if num < len(ENDPOINTS) or formats:
continue
- if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
- error_info = self._parse_json(ee.cause.read().decode(), item_id, fatal=False)
+ if isinstance(ee.cause, HTTPError) and ee.cause.status == 404:
+ error_info = self._parse_json(ee.cause.response.read().decode(), item_id, fatal=False)
if error_info:
raise ExtractorError(
'%s said: %s' % (self.IE_NAME, error_info['message']),
diff --git a/yt_dlp/extractor/rts.py b/yt_dlp/extractor/rts.py
index 81c4d7cac..9f73d1811 100644
--- a/yt_dlp/extractor/rts.py
+++ b/yt_dlp/extractor/rts.py
@@ -136,8 +136,8 @@ class RTSIE(SRGSSRIE): # XXX: Do not subclass from concrete IE
if not entries:
page, urlh = self._download_webpage_handle(url, display_id)
- if re.match(self._VALID_URL, urlh.geturl()).group('id') != media_id:
- return self.url_result(urlh.geturl(), 'RTS')
+ if re.match(self._VALID_URL, urlh.url).group('id') != media_id:
+ return self.url_result(urlh.url, 'RTS')
# article with videos on rhs
videos = re.findall(
diff --git a/yt_dlp/extractor/rumble.py b/yt_dlp/extractor/rumble.py
index 82f3f0f8c..f8bf4a182 100644
--- a/yt_dlp/extractor/rumble.py
+++ b/yt_dlp/extractor/rumble.py
@@ -2,7 +2,7 @@ import itertools
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
UnsupportedError,
@@ -371,7 +371,7 @@ class RumbleChannelIE(InfoExtractor):
try:
webpage = self._download_webpage(f'{url}?page={page}', playlist_id, note='Downloading page %d' % page)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 404:
break
raise
for video_url in re.findall(r'class=video-item--a\s?href=([^>]+\.html)', webpage):
diff --git a/yt_dlp/extractor/safari.py b/yt_dlp/extractor/safari.py
index 450a661e9..8d322d710 100644
--- a/yt_dlp/extractor/safari.py
+++ b/yt_dlp/extractor/safari.py
@@ -28,13 +28,13 @@ class SafariBaseIE(InfoExtractor):
'Downloading login page')
def is_logged(urlh):
- return 'learning.oreilly.com/home/' in urlh.geturl()
+ return 'learning.oreilly.com/home/' in urlh.url
if is_logged(urlh):
self.LOGGED_IN = True
return
- redirect_url = urlh.geturl()
+ redirect_url = urlh.url
parsed_url = compat_urlparse.urlparse(redirect_url)
qs = compat_parse_qs(parsed_url.query)
next_uri = compat_urlparse.urljoin(
@@ -129,7 +129,7 @@ class SafariIE(SafariBaseIE):
webpage, urlh = self._download_webpage_handle(url, video_id)
- mobj = re.match(self._VALID_URL, urlh.geturl())
+ mobj = re.match(self._VALID_URL, urlh.url)
reference_id = mobj.group('reference_id')
if not reference_id:
reference_id = self._search_regex(
diff --git a/yt_dlp/extractor/sbs.py b/yt_dlp/extractor/sbs.py
index 119106e8e..7a9115047 100644
--- a/yt_dlp/extractor/sbs.py
+++ b/yt_dlp/extractor/sbs.py
@@ -1,6 +1,6 @@
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
float_or_none,
int_or_none,
parse_duration,
diff --git a/yt_dlp/extractor/sevenplus.py b/yt_dlp/extractor/sevenplus.py
index 222bf6ce7..6c688d150 100644
--- a/yt_dlp/extractor/sevenplus.py
+++ b/yt_dlp/extractor/sevenplus.py
@@ -2,10 +2,8 @@ import json
import re
from .brightcove import BrightcoveNewBaseIE
-from ..compat import (
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
try_get,
@@ -97,9 +95,9 @@ class SevenPlusIE(BrightcoveNewBaseIE):
'videoType': 'vod',
}, headers=headers)['media']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
raise ExtractorError(self._parse_json(
- e.cause.read().decode(), episode_id)[0]['error_code'], expected=True)
+ e.cause.response.read().decode(), episode_id)[0]['error_code'], expected=True)
raise
for source in media.get('sources', {}):
diff --git a/yt_dlp/extractor/shahid.py b/yt_dlp/extractor/shahid.py
index 26a0bff40..d509e8879 100644
--- a/yt_dlp/extractor/shahid.py
+++ b/yt_dlp/extractor/shahid.py
@@ -3,7 +3,7 @@ import math
import re
from .aws import AWSIE
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
ExtractorError,
@@ -22,7 +22,7 @@ class ShahidBaseIE(AWSIE):
def _handle_error(self, e):
fail_data = self._parse_json(
- e.cause.read().decode('utf-8'), None, fatal=False)
+ e.cause.response.read().decode('utf-8'), None, fatal=False)
if fail_data:
faults = fail_data.get('faults', [])
faults_message = ', '.join([clean_html(fault['userMessage']) for fault in faults if fault.get('userMessage')])
@@ -40,7 +40,7 @@ class ShahidBaseIE(AWSIE):
'secret_key': '4WUUJWuFvtTkXbhaWTDv7MhO+0LqoYDWfEnUXoWn',
}, video_id, query)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
+ if isinstance(e.cause, HTTPError):
self._handle_error(e)
raise
@@ -88,7 +88,7 @@ class ShahidIE(ShahidBaseIE):
'Content-Type': 'application/json; charset=UTF-8',
})['user']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
+ if isinstance(e.cause, HTTPError):
self._handle_error(e)
raise
diff --git a/yt_dlp/extractor/sina.py b/yt_dlp/extractor/sina.py
index aeba4e377..984281188 100644
--- a/yt_dlp/extractor/sina.py
+++ b/yt_dlp/extractor/sina.py
@@ -1,12 +1,12 @@
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
ExtractorError,
+ clean_html,
+ get_element_by_attribute,
int_or_none,
- update_url_query,
qualities,
- get_element_by_attribute,
- clean_html,
+ update_url_query,
)
@@ -60,7 +60,7 @@ class SinaIE(InfoExtractor):
self.to_screen('Getting video id')
request = HEADRequest(url)
_, urlh = self._download_webpage_handle(request, 'NA', False)
- return self._real_extract(urlh.geturl())
+ return self._real_extract(urlh.url)
else:
pseudo_id = mobj.group('pseudo_id')
webpage = self._download_webpage(url, pseudo_id)
diff --git a/yt_dlp/extractor/sixplay.py b/yt_dlp/extractor/sixplay.py
index a6fb6c1f5..ef93b9276 100644
--- a/yt_dlp/extractor/sixplay.py
+++ b/yt_dlp/extractor/sixplay.py
@@ -79,7 +79,7 @@ class SixPlayIE(InfoExtractor):
headers=self.geo_verification_headers())
if not urlh:
continue
- asset_url = urlh.geturl()
+ asset_url = urlh.url
asset_url = asset_url.replace('_drmnp.ism/', '_unpnp.ism/')
for i in range(3, 0, -1):
asset_url = asset_url = asset_url.replace('_sd1/', '_sd%d/' % i)
diff --git a/yt_dlp/extractor/slideslive.py b/yt_dlp/extractor/slideslive.py
index 3d36edbbc..25f867a60 100644
--- a/yt_dlp/extractor/slideslive.py
+++ b/yt_dlp/extractor/slideslive.py
@@ -426,7 +426,7 @@ class SlidesLiveIE(InfoExtractor):
video_id, headers=traverse_obj(parse_qs(url), {
'Referer': ('embed_parent_url', -1),
'Origin': ('embed_container_origin', -1)}))
- redirect_url = urlh.geturl()
+ redirect_url = urlh.url
if 'domain_not_allowed' in redirect_url:
domain = traverse_obj(parse_qs(redirect_url), ('allowed_domains[]', ...), get_all=False)
if not domain:
diff --git a/yt_dlp/extractor/sonyliv.py b/yt_dlp/extractor/sonyliv.py
index 5ebe20df7..437957259 100644
--- a/yt_dlp/extractor/sonyliv.py
+++ b/yt_dlp/extractor/sonyliv.py
@@ -6,7 +6,7 @@ import time
import uuid
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -123,12 +123,12 @@ class SonyLIVIE(InfoExtractor):
'https://apiv2.sonyliv.com/AGL/%s/A/ENG/WEB/%s' % (version, path),
video_id, headers=self._HEADERS)['resultObj']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 406 and self._parse_json(
- e.cause.read().decode(), video_id)['message'] == 'Please subscribe to watch this content':
+ if isinstance(e.cause, HTTPError) and e.cause.status == 406 and self._parse_json(
+ e.cause.response.read().decode(), video_id)['message'] == 'Please subscribe to watch this content':
self.raise_login_required(self._LOGIN_HINT, method=None)
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
message = self._parse_json(
- e.cause.read().decode(), video_id)['message']
+ e.cause.response.read().decode(), video_id)['message']
if message == 'Geoblocked Country':
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
raise ExtractorError(message)
diff --git a/yt_dlp/extractor/soundcloud.py b/yt_dlp/extractor/soundcloud.py
index 979f23f44..a7c2afd49 100644
--- a/yt_dlp/extractor/soundcloud.py
+++ b/yt_dlp/extractor/soundcloud.py
@@ -7,15 +7,13 @@ from .common import (
InfoExtractor,
SearchInfoExtractor
)
-from ..compat import (
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_str
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import HTTPError
from ..utils import (
error_to_compat_str,
ExtractorError,
float_or_none,
- HEADRequest,
int_or_none,
KNOWN_EXTENSIONS,
mimetype2ext,
@@ -26,7 +24,6 @@ from ..utils import (
update_url_query,
url_or_none,
urlhandle_detect_ext,
- sanitized_Request,
)
@@ -103,7 +100,7 @@ class SoundcloudBaseIE(InfoExtractor):
try:
return super()._download_json(*args, **kwargs)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (401, 403):
+ if isinstance(e.cause, HTTPError) and e.cause.status in (401, 403):
self._store_client_id(None)
self._update_client_id()
continue
@@ -123,7 +120,7 @@ class SoundcloudBaseIE(InfoExtractor):
self._access_token = password
query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID
payload = {'session': {'access_token': self._access_token}}
- token_verification = sanitized_Request(self._API_VERIFY_AUTH_TOKEN % query, json.dumps(payload).encode('utf-8'))
+ token_verification = Request(self._API_VERIFY_AUTH_TOKEN % query, json.dumps(payload).encode('utf-8'))
response = self._download_json(token_verification, None, note='Verifying login token...', fatal=False)
if response is not False:
self._HEADERS = {'Authorization': 'OAuth ' + self._access_token}
@@ -212,7 +209,7 @@ class SoundcloudBaseIE(InfoExtractor):
urlh = self._request_webpage(
HEADRequest(redirect_url), track_id, fatal=False)
if urlh:
- format_url = urlh.geturl()
+ format_url = urlh.url
format_urls.add(format_url)
formats.append({
'format_id': 'download',
@@ -669,7 +666,7 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudBaseIE):
except ExtractorError as e:
# Downloading page may result in intermittent 502 HTTP error
# See https://github.com/yt-dlp/yt-dlp/issues/872
- if not isinstance(e.cause, compat_HTTPError) or e.cause.code != 502:
+ if not isinstance(e.cause, HTTPError) or e.cause.status != 502:
raise
retry.error = e
continue
diff --git a/yt_dlp/extractor/teachable.py b/yt_dlp/extractor/teachable.py
index c212a4926..01906bda9 100644
--- a/yt_dlp/extractor/teachable.py
+++ b/yt_dlp/extractor/teachable.py
@@ -56,7 +56,7 @@ class TeachableBaseIE(InfoExtractor):
self._logged_in = True
return
- login_url = urlh.geturl()
+ login_url = urlh.url
login_form = self._hidden_inputs(login_page)
diff --git a/yt_dlp/extractor/telemundo.py b/yt_dlp/extractor/telemundo.py
index 88f29cb83..54e74a6c0 100644
--- a/yt_dlp/extractor/telemundo.py
+++ b/yt_dlp/extractor/telemundo.py
@@ -1,9 +1,6 @@
from .common import InfoExtractor
-from ..utils import (
- try_get,
- unified_timestamp,
- HEADRequest,
-)
+from ..networking import HEADRequest
+from ..utils import try_get, unified_timestamp
class TelemundoIE(InfoExtractor):
@@ -38,7 +35,7 @@ class TelemundoIE(InfoExtractor):
m3u8_url = self._request_webpage(HEADRequest(
redirect_url + '?format=redirect&manifest=m3u&format=redirect&Tracking=true&Embedded=true&formats=MPEG4'),
- video_id, 'Processing m3u8').geturl()
+ video_id, 'Processing m3u8').url
formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
date = unified_timestamp(try_get(
metadata, lambda x: x['props']['initialState']['video']['associatedPlaylists'][0]['videos'][0]['datePublished'].split(' ', 1)[1]))
diff --git a/yt_dlp/extractor/tennistv.py b/yt_dlp/extractor/tennistv.py
index bc64226bf..c1b4a3312 100644
--- a/yt_dlp/extractor/tennistv.py
+++ b/yt_dlp/extractor/tennistv.py
@@ -86,7 +86,7 @@ class TennisTVIE(InfoExtractor):
})
self.get_token(None, {
- 'code': urllib.parse.parse_qs(handle.geturl())['code'][-1],
+ 'code': urllib.parse.parse_qs(handle.url)['code'][-1],
'grant_type': 'authorization_code',
'client_id': 'tennis-tv-web',
'redirect_uri': 'https://www.tennistv.com/resources/v1.1.10/html/silent-check-sso.html'
diff --git a/yt_dlp/extractor/tenplay.py b/yt_dlp/extractor/tenplay.py
index 633032e31..c7097cf02 100644
--- a/yt_dlp/extractor/tenplay.py
+++ b/yt_dlp/extractor/tenplay.py
@@ -2,11 +2,8 @@ from datetime import datetime
import base64
from .common import InfoExtractor
-from ..utils import (
- HEADRequest,
- int_or_none,
- urlencode_postdata,
-)
+from ..networking import HEADRequest
+from ..utils import int_or_none, urlencode_postdata
class TenPlayIE(InfoExtractor):
@@ -94,7 +91,7 @@ class TenPlayIE(InfoExtractor):
data.get('playbackApiEndpoint'), content_id, 'Downloading video JSON',
headers=headers).get('source')
m3u8_url = self._request_webpage(HEADRequest(
- _video_url), content_id).geturl()
+ _video_url), content_id).url
if '10play-not-in-oz' in m3u8_url:
self.raise_geo_restricted(countries=['AU'])
formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4')
diff --git a/yt_dlp/extractor/tfo.py b/yt_dlp/extractor/tfo.py
index a24789cb3..d417f50e1 100644
--- a/yt_dlp/extractor/tfo.py
+++ b/yt_dlp/extractor/tfo.py
@@ -1,12 +1,8 @@
import json
from .common import InfoExtractor
-from ..utils import (
- HEADRequest,
- ExtractorError,
- int_or_none,
- clean_html,
-)
+from ..networking import HEADRequest
+from ..utils import ExtractorError, clean_html, int_or_none
class TFOIE(InfoExtractor):
diff --git a/yt_dlp/extractor/theplatform.py b/yt_dlp/extractor/theplatform.py
index e659b8ee1..537f6f6cd 100644
--- a/yt_dlp/extractor/theplatform.py
+++ b/yt_dlp/extractor/theplatform.py
@@ -7,13 +7,13 @@ import hashlib
from .once import OnceIE
from .adobepass import AdobePassIE
+from ..networking import Request
from ..utils import (
determine_ext,
ExtractorError,
float_or_none,
int_or_none,
parse_qs,
- sanitized_Request,
unsmuggle_url,
update_url_query,
xpath_with_ns,
@@ -270,7 +270,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE):
source_url = smuggled_data.get('source_url')
if source_url:
headers['Referer'] = source_url
- request = sanitized_Request(url, headers=headers)
+ request = Request(url, headers=headers)
webpage = self._download_webpage(request, video_id)
smil_url = self._search_regex(
r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
diff --git a/yt_dlp/extractor/thisoldhouse.py b/yt_dlp/extractor/thisoldhouse.py
index 55b6413ae..cc7beeea5 100644
--- a/yt_dlp/extractor/thisoldhouse.py
+++ b/yt_dlp/extractor/thisoldhouse.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..utils import HEADRequest
+from ..networking import HEADRequest
class ThisOldHouseIE(InfoExtractor):
@@ -50,6 +50,6 @@ class ThisOldHouseIE(InfoExtractor):
r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})[^\'"]*)[\'"]',
webpage, 'video url')
if 'subscription_required=true' in video_url or 'c-entry-group-labels__image' in webpage:
- return self.url_result(self._request_webpage(HEADRequest(video_url), display_id).geturl(), 'Zype', display_id)
+ return self.url_result(self._request_webpage(HEADRequest(video_url), display_id).url, 'Zype', display_id)
video_id = self._search_regex(r'(?:https?:)?//(?:www\.)?thisoldhouse\.(?:chorus\.build|com)/videos/zype/([0-9a-f]{24})', video_url, 'video id')
return self.url_result(self._ZYPE_TMPL % video_id, 'Zype', video_id)
diff --git a/yt_dlp/extractor/threeqsdn.py b/yt_dlp/extractor/threeqsdn.py
index b1041902b..7841f8da6 100644
--- a/yt_dlp/extractor/threeqsdn.py
+++ b/yt_dlp/extractor/threeqsdn.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -90,7 +90,7 @@ class ThreeQSDNIE(InfoExtractor):
config = self._download_json(
url.replace('://playout.3qsdn.com/', '://playout.3qsdn.com/config/'), video_id)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
self.raise_geo_restricted()
raise
diff --git a/yt_dlp/extractor/tiktok.py b/yt_dlp/extractor/tiktok.py
index 2f491c317..48de61f93 100644
--- a/yt_dlp/extractor/tiktok.py
+++ b/yt_dlp/extractor/tiktok.py
@@ -7,9 +7,9 @@ import time
from .common import InfoExtractor
from ..compat import compat_urllib_parse_unquote, compat_urllib_parse_urlparse
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
LazyList,
UnsupportedError,
UserNotLive,
@@ -1084,7 +1084,7 @@ class TikTokVMIE(InfoExtractor):
def _real_extract(self, url):
new_url = self._request_webpage(
- HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).geturl()
+ HEADRequest(url), self._match_id(url), headers={'User-Agent': 'facebookexternalhit/1.1'}).url
if self.suitable(new_url): # Prevent infinite loop in case redirect fails
raise UnsupportedError(new_url)
return self.url_result(new_url)
diff --git a/yt_dlp/extractor/toutv.py b/yt_dlp/extractor/toutv.py
index f60c199f0..ced1224fa 100644
--- a/yt_dlp/extractor/toutv.py
+++ b/yt_dlp/extractor/toutv.py
@@ -1,7 +1,7 @@
import json
from .radiocanada import RadioCanadaIE
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -52,8 +52,8 @@ class TouTvIE(RadioCanadaIE): # XXX: Do not subclass from concrete IE
'Content-Type': 'application/json;charset=utf-8',
})['access_token']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read().decode(), None)['Message']
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read().decode(), None)['Message']
raise ExtractorError(error, expected=True)
raise
self._claims = self._call_api('validation/v2/getClaims')['claims']
diff --git a/yt_dlp/extractor/triller.py b/yt_dlp/extractor/triller.py
index 6a4dadb9b..c5d01c827 100644
--- a/yt_dlp/extractor/triller.py
+++ b/yt_dlp/extractor/triller.py
@@ -3,9 +3,9 @@ import json
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
ExtractorError,
- HEADRequest,
UnsupportedError,
determine_ext,
int_or_none,
@@ -327,7 +327,7 @@ class TrillerShortIE(InfoExtractor):
}]
def _real_extract(self, url):
- real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).geturl()
+ real_url = self._request_webpage(HEADRequest(url), self._match_id(url)).url
if self.suitable(real_url): # Prevent infinite loop in case redirect fails
raise UnsupportedError(real_url)
return self.url_result(real_url)
diff --git a/yt_dlp/extractor/trueid.py b/yt_dlp/extractor/trueid.py
index 696343627..86f0990e8 100644
--- a/yt_dlp/extractor/trueid.py
+++ b/yt_dlp/extractor/trueid.py
@@ -1,5 +1,5 @@
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -88,9 +88,9 @@ class TrueIDIE(InfoExtractor):
stream_data = self._download_json(
f'https://{domain}/cmsPostProxy/contents/video/{video_id}/streamer?os=android', video_id, data=b'')['data']
except ExtractorError as e:
- if not isinstance(e.cause, compat_HTTPError):
+ if not isinstance(e.cause, HTTPError):
raise e
- errmsg = self._parse_json(e.cause.read().decode(), video_id)['meta']['message']
+ errmsg = self._parse_json(e.cause.response.read().decode(), video_id)['meta']['message']
if 'country' in errmsg:
self.raise_geo_restricted(
errmsg, [initial_data['display_country']] if initial_data.get('display_country') else None, True)
diff --git a/yt_dlp/extractor/tubetugraz.py b/yt_dlp/extractor/tubetugraz.py
index 2199fea19..a351e4e55 100644
--- a/yt_dlp/extractor/tubetugraz.py
+++ b/yt_dlp/extractor/tubetugraz.py
@@ -22,7 +22,7 @@ class TubeTuGrazBaseIE(InfoExtractor):
return
content, urlh = self._download_webpage_handle(
- urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+ urlh.url, None, fatal=False, headers={'referer': urlh.url},
note='logging in', errnote='unable to log in',
data=urlencode_postdata({
'lang': 'de',
@@ -30,7 +30,7 @@ class TubeTuGrazBaseIE(InfoExtractor):
'j_username': username,
'j_password': password
}))
- if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+ if not urlh or urlh.url == 'https://tube.tugraz.at/paella/ui/index.html':
return
if not self._html_search_regex(
@@ -40,14 +40,14 @@ class TubeTuGrazBaseIE(InfoExtractor):
return
content, urlh = self._download_webpage_handle(
- urlh.geturl(), None, fatal=False, headers={'referer': urlh.geturl()},
+ urlh.url, None, fatal=False, headers={'referer': urlh.url},
note='logging in with TFA', errnote='unable to log in with TFA',
data=urlencode_postdata({
'lang': 'de',
'_eventId_proceed': '',
'j_tokenNumber': self._get_tfa_info(),
}))
- if not urlh or urlh.geturl() == 'https://tube.tugraz.at/paella/ui/index.html':
+ if not urlh or urlh.url == 'https://tube.tugraz.at/paella/ui/index.html':
return
self.report_warning('unable to login: incorrect TFA code')
diff --git a/yt_dlp/extractor/tubitv.py b/yt_dlp/extractor/tubitv.py
index de8b5da69..bd46bc363 100644
--- a/yt_dlp/extractor/tubitv.py
+++ b/yt_dlp/extractor/tubitv.py
@@ -1,13 +1,13 @@
import re
from .common import InfoExtractor
+from ..networking import Request
from ..utils import (
ExtractorError,
int_or_none,
js_to_json,
- sanitized_Request,
- urlencode_postdata,
traverse_obj,
+ urlencode_postdata,
)
@@ -72,8 +72,8 @@ class TubiTvIE(InfoExtractor):
'password': password,
}
payload = urlencode_postdata(form_data)
- request = sanitized_Request(self._LOGIN_URL, payload)
- request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ request = Request(self._LOGIN_URL, payload)
+ request.headers['Content-Type'] = 'application/x-www-form-urlencoded'
login_page = self._download_webpage(
request, None, False, 'Wrong login info')
if not re.search(r'id="tubi-logout"', login_page):
diff --git a/yt_dlp/extractor/tumblr.py b/yt_dlp/extractor/tumblr.py
index 88d4ae32d..a26bdcaae 100644
--- a/yt_dlp/extractor/tumblr.py
+++ b/yt_dlp/extractor/tumblr.py
@@ -274,7 +274,7 @@ class TumblrIE(InfoExtractor):
url = f'http://{blog}.tumblr.com/post/{video_id}/'
webpage, urlh = self._download_webpage_handle(url, video_id)
- redirect_url = urlh.geturl()
+ redirect_url = urlh.url
api_only = bool(self._search_regex(
r'(tumblr.com|^)/(safe-mode|login_required|blog/view)',
diff --git a/yt_dlp/extractor/tunein.py b/yt_dlp/extractor/tunein.py
index e02121bd8..fd2fe132c 100644
--- a/yt_dlp/extractor/tunein.py
+++ b/yt_dlp/extractor/tunein.py
@@ -225,10 +225,10 @@ class TuneInShortenerIE(InfoExtractor):
urlh = self._request_webpage(
url, redirect_id, note='Downloading redirect page')
- url = urlh.geturl()
+ url = urlh.url
url_parsed = urllib.parse.urlparse(url)
if url_parsed.port == 443:
- url = url_parsed._replace(netloc=url_parsed.hostname).geturl()
+ url = url_parsed._replace(netloc=url_parsed.hostname).url
self.to_screen('Following redirect: %s' % url)
return self.url_result(url)
diff --git a/yt_dlp/extractor/tv2.py b/yt_dlp/extractor/tv2.py
index c51e63371..f6b452dc8 100644
--- a/yt_dlp/extractor/tv2.py
+++ b/yt_dlp/extractor/tv2.py
@@ -1,7 +1,7 @@
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -57,8 +57,8 @@ class TV2IE(InfoExtractor):
headers={'content-type': 'application/json'},
data='{"device":{"id":"1-1-1","name":"Nettleser (HTML)"}}'.encode())['playback']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read().decode(), video_id)['error']
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read().decode(), video_id)['error']
error_code = error.get('code')
if error_code == 'ASSET_PLAYBACK_INVALID_GEO_LOCATION':
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
@@ -211,8 +211,8 @@ class KatsomoIE(InfoExtractor):
api_base + '/play.json?protocol=%s&videoFormat=SMIL+ISMUSP' % protocol,
video_id, 'Downloading play JSON')['playback']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- error = self._parse_json(e.cause.read().decode(), video_id)['error']
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ error = self._parse_json(e.cause.response.read().decode(), video_id)['error']
error_code = error.get('code')
if error_code == 'ASSET_PLAYBACK_INVALID_GEO_LOCATION':
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
diff --git a/yt_dlp/extractor/tvp.py b/yt_dlp/extractor/tvp.py
index c686044fa..2aa0dd870 100644
--- a/yt_dlp/extractor/tvp.py
+++ b/yt_dlp/extractor/tvp.py
@@ -488,9 +488,9 @@ class TVPVODBaseIE(InfoExtractor):
f'{self._API_BASE_URL}/{resource}', video_id,
query={'lang': 'pl', 'platform': 'BROWSER', **query},
expected_status=lambda x: is_valid(x) or 400 <= x < 500, **kwargs)
- if is_valid(urlh.getcode()):
+ if is_valid(urlh.status):
return document
- raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.getcode()})')
+ raise ExtractorError(f'Woronicza said: {document.get("code")} (HTTP {urlh.status})')
def _parse_video(self, video, with_url=True):
info_dict = traverse_obj(video, {
diff --git a/yt_dlp/extractor/tvplay.py b/yt_dlp/extractor/tvplay.py
index e056f9872..48a6efe1c 100644
--- a/yt_dlp/extractor/tvplay.py
+++ b/yt_dlp/extractor/tvplay.py
@@ -1,10 +1,8 @@
import re
from .common import InfoExtractor
-from ..compat import (
- compat_HTTPError,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
+from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
@@ -129,8 +127,8 @@ class TVPlayIE(InfoExtractor):
'http://playapi.mtgx.tv/v3/videos/stream/%s' % video_id,
video_id, 'Downloading streams JSON')
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- msg = self._parse_json(e.cause.read().decode('utf-8'), video_id)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ msg = self._parse_json(e.cause.response.read().decode('utf-8'), video_id)
raise ExtractorError(msg['msg'], expected=True)
raise
diff --git a/yt_dlp/extractor/tvplayer.py b/yt_dlp/extractor/tvplayer.py
index b05355f87..228c2366e 100644
--- a/yt_dlp/extractor/tvplayer.py
+++ b/yt_dlp/extractor/tvplayer.py
@@ -1,8 +1,6 @@
from .common import InfoExtractor
-from ..compat import (
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
extract_attributes,
try_get,
@@ -64,9 +62,9 @@ class TVPlayerIE(InfoExtractor):
'validate': validate,
}))['tvplayer']['response']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
+ if isinstance(e.cause, HTTPError):
response = self._parse_json(
- e.cause.read().decode(), resource_id)['tvplayer']['response']
+ e.cause.response.read().decode(), resource_id)['tvplayer']['response']
raise ExtractorError(
'%s said: %s' % (self.IE_NAME, response['error']), expected=True)
raise
diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 2548dae04..dff353a4f 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -107,9 +107,9 @@ class TwitCastingIE(InfoExtractor):
url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'},
note='Trying video password')
- if urlh.geturl() != url and request_data:
+ if urlh.url != url and request_data:
webpage = self._download_webpage(
- urlh.geturl(), video_id, data=request_data,
+ urlh.url, video_id, data=request_data,
headers={'Origin': 'https://twitcasting.tv'},
note='Retrying authentication')
# has to check here as the first request can contain password input form even if the password is correct
diff --git a/yt_dlp/extractor/twitch.py b/yt_dlp/extractor/twitch.py
index c8ee52014..3297ef091 100644
--- a/yt_dlp/extractor/twitch.py
+++ b/yt_dlp/extractor/twitch.py
@@ -71,7 +71,7 @@ class TwitchBaseIE(InfoExtractor):
form = self._hidden_inputs(page)
form.update(data)
- page_url = urlh.geturl()
+ page_url = urlh.url
post_url = self._search_regex(
r'<form[^>]+action=(["\'])(?P<url>.+?)\1', page,
'post url', default=self._LOGIN_POST_URL, group='url')
diff --git a/yt_dlp/extractor/twitter.py b/yt_dlp/extractor/twitter.py
index fc157ac22..4015277a8 100644
--- a/yt_dlp/extractor/twitter.py
+++ b/yt_dlp/extractor/twitter.py
@@ -1596,7 +1596,7 @@ class TwitterShortenerIE(TwitterBaseIE):
if eid:
id = eid
url = self._BASE_URL + id
- new_url = self._request_webpage(url, id, headers={'User-Agent': 'curl'}).geturl()
+ new_url = self._request_webpage(url, id, headers={'User-Agent': 'curl'}).url
__UNSAFE_LINK = "https://twitter.com/safety/unsafe_link_warning?unsafe_link="
if new_url.startswith(__UNSAFE_LINK):
new_url = new_url.replace(__UNSAFE_LINK, "")
diff --git a/yt_dlp/extractor/udemy.py b/yt_dlp/extractor/udemy.py
index 329e5da2d..5c296051a 100644
--- a/yt_dlp/extractor/udemy.py
+++ b/yt_dlp/extractor/udemy.py
@@ -1,8 +1,9 @@
import re
-import urllib.request
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str, compat_urlparse
+from ..compat import compat_str, compat_urlparse
+from ..networking import Request
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
@@ -10,7 +11,6 @@ from ..utils import (
float_or_none,
int_or_none,
js_to_json,
- sanitized_Request,
smuggle_url,
try_get,
unescapeHTML,
@@ -153,11 +153,10 @@ class UdemyIE(InfoExtractor):
headers['X-Udemy-Bearer-Token'] = cookie.value
headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
- if isinstance(url_or_request, urllib.request.Request):
- for header, value in headers.items():
- url_or_request.add_header(header, value)
+ if isinstance(url_or_request, Request):
+ url_or_request.headers.update(headers)
else:
- url_or_request = sanitized_Request(url_or_request, headers=headers)
+ url_or_request = Request(url_or_request, headers=headers)
response = super(UdemyIE, self)._download_json(url_or_request, *args, **kwargs)
self._handle_error(response)
@@ -212,7 +211,7 @@ class UdemyIE(InfoExtractor):
lecture = self._download_lecture(course_id, lecture_id)
except ExtractorError as e:
# Error could possibly mean we are not enrolled in the course
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
webpage = webpage or self._download_webpage(url, lecture_id)
self._enroll_course(url, webpage, course_id)
lecture = self._download_lecture(course_id, lecture_id)
diff --git a/yt_dlp/extractor/vevo.py b/yt_dlp/extractor/vevo.py
index da4ce49ca..aa40227a7 100644
--- a/yt_dlp/extractor/vevo.py
+++ b/yt_dlp/extractor/vevo.py
@@ -2,10 +2,8 @@ import re
import json
from .common import InfoExtractor
-from ..compat import (
- compat_str,
- compat_HTTPError,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -184,8 +182,8 @@ class VevoIE(VevoBaseIE):
try:
data = self._download_json(self._api_url_template % path, *args, **kwargs)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
- errors = self._parse_json(e.cause.read().decode(), None)['errors']
+ if isinstance(e.cause, HTTPError):
+ errors = self._parse_json(e.cause.response.read().decode(), None)['errors']
error_message = ', '.join([error['message'] for error in errors])
raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
raise
diff --git a/yt_dlp/extractor/vice.py b/yt_dlp/extractor/vice.py
index d1a3b48aa..8a7126853 100644
--- a/yt_dlp/extractor/vice.py
+++ b/yt_dlp/extractor/vice.py
@@ -7,10 +7,8 @@ import time
from .adobepass import AdobePassIE
from .common import InfoExtractor
from .youtube import YoutubeIE
-from ..compat import (
- compat_HTTPError,
- compat_str,
-)
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
ExtractorError,
@@ -140,8 +138,8 @@ class ViceIE(ViceBaseIE, AdobePassIE):
'https://vms.vice.com/%s/video/preplay/%s' % (locale, video_id),
video_id, query=query)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401):
- error = json.loads(e.cause.read().decode())
+ if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401):
+ error = json.loads(e.cause.response.read().decode())
error_message = error.get('error_description') or error['details']
raise ExtractorError('%s said: %s' % (
self.IE_NAME, error_message), expected=True)
diff --git a/yt_dlp/extractor/videocampus_sachsen.py b/yt_dlp/extractor/videocampus_sachsen.py
index 982ab3dd0..37bc7d718 100644
--- a/yt_dlp/extractor/videocampus_sachsen.py
+++ b/yt_dlp/extractor/videocampus_sachsen.py
@@ -2,7 +2,7 @@ import functools
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import ExtractorError, OnDemandPagedList, urlencode_postdata
@@ -169,7 +169,7 @@ class VideocampusSachsenIE(InfoExtractor):
f'https://{host}/media/hlsMedium/key/{video_id}/format/auto/ext/mp4/learning/0/path/m3u8',
video_id, 'mp4', m3u8_id='hls', fatal=True)
except ExtractorError as e:
- if not isinstance(e.cause, compat_HTTPError) or e.cause.code not in (404, 500):
+ if not isinstance(e.cause, HTTPError) or e.cause.status not in (404, 500):
raise
formats.append({'url': f'https://{host}/getMedium/{video_id}.mp4'})
diff --git a/yt_dlp/extractor/vidio.py b/yt_dlp/extractor/vidio.py
index 23e1aaf20..770aa284d 100644
--- a/yt_dlp/extractor/vidio.py
+++ b/yt_dlp/extractor/vidio.py
@@ -39,7 +39,7 @@ class VidioBaseIE(InfoExtractor):
login_post, login_post_urlh = self._download_webpage_handle(
self._LOGIN_URL, None, 'Logging in', data=urlencode_postdata(login_form), expected_status=[302, 401])
- if login_post_urlh.getcode() == 401:
+ if login_post_urlh.status == 401:
if get_element_by_class('onboarding-content-register-popup__title', login_post):
raise ExtractorError(
'Unable to log in: The provided email has not registered yet.', expected=True)
diff --git a/yt_dlp/extractor/vidlii.py b/yt_dlp/extractor/vidlii.py
index cde4274d9..44353b7fc 100644
--- a/yt_dlp/extractor/vidlii.py
+++ b/yt_dlp/extractor/vidlii.py
@@ -1,8 +1,8 @@
import re
from .common import InfoExtractor
+from ..networking import HEADRequest
from ..utils import (
- HEADRequest,
format_field,
float_or_none,
get_element_by_id,
diff --git a/yt_dlp/extractor/viewlift.py b/yt_dlp/extractor/viewlift.py
index 381260114..8f686f05d 100644
--- a/yt_dlp/extractor/viewlift.py
+++ b/yt_dlp/extractor/viewlift.py
@@ -1,7 +1,7 @@
import json
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -46,8 +46,8 @@ class ViewLiftBaseIE(InfoExtractor):
return self._download_json(
self._API_BASE + path, video_id, headers={'Authorization': self._TOKENS.get(site)}, query=query)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
- webpage = e.cause.read().decode()
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
+ webpage = e.cause.response.read().decode()
try:
error_message = traverse_obj(json.loads(webpage), 'errorMessage', 'message')
except json.JSONDecodeError:
diff --git a/yt_dlp/extractor/viidea.py b/yt_dlp/extractor/viidea.py
index 4cdf2677b..649ffe395 100644
--- a/yt_dlp/extractor/viidea.py
+++ b/yt_dlp/extractor/viidea.py
@@ -2,10 +2,10 @@ import re
from .common import InfoExtractor
from ..compat import (
- compat_HTTPError,
compat_str,
compat_urlparse,
)
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
js_to_json,
@@ -133,9 +133,9 @@ class ViideaIE(InfoExtractor):
'%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
lecture_id)['lecture'][0]
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 403:
msg = self._parse_json(
- e.cause.read().decode('utf-8'), lecture_id)
+ e.cause.response.read().decode('utf-8'), lecture_id)
raise ExtractorError(msg['detail'], expected=True)
raise
diff --git a/yt_dlp/extractor/vimeo.py b/yt_dlp/extractor/vimeo.py
index d81d9c551..e72fa50fa 100644
--- a/yt_dlp/extractor/vimeo.py
+++ b/yt_dlp/extractor/vimeo.py
@@ -2,20 +2,16 @@ import base64
import functools
import re
import itertools
-import urllib.error
from .common import InfoExtractor
-from ..compat import (
- compat_HTTPError,
- compat_str,
- compat_urlparse,
-)
+from ..compat import compat_str, compat_urlparse
+from ..networking import HEADRequest, Request
+from ..networking.exceptions import HTTPError
from ..utils import (
clean_html,
determine_ext,
ExtractorError,
get_element_by_class,
- HEADRequest,
js_to_json,
int_or_none,
merge_dicts,
@@ -23,7 +19,6 @@ from ..utils import (
parse_filesize,
parse_iso8601,
parse_qs,
- sanitized_Request,
smuggle_url,
str_or_none,
try_get,
@@ -72,7 +67,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
'Referer': self._LOGIN_URL,
})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 418:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 418:
raise ExtractorError(
'Unable to log in: bad username or password',
expected=True)
@@ -809,7 +804,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
'X-Requested-With': 'XMLHttpRequest',
})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
raise ExtractorError('Wrong password', expected=True)
raise
@@ -832,10 +827,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
# Retrieve video webpage to extract further information
webpage, urlh = self._download_webpage_handle(
url, video_id, headers=headers)
- redirect_url = urlh.geturl()
+ redirect_url = urlh.url
except ExtractorError as ee:
- if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
- errmsg = ee.cause.read()
+ if isinstance(ee.cause, HTTPError) and ee.cause.status == 403:
+ errmsg = ee.cause.response.read()
if b'Because of its privacy settings, this video cannot be played here' in errmsg:
raise ExtractorError(
'Cannot download embed-only video without embedding '
@@ -1154,7 +1149,7 @@ class VimeoAlbumIE(VimeoBaseInfoExtractor):
'Authorization': 'jwt ' + authorization,
})['data']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
return
for video in videos:
link = video.get('link')
@@ -1196,7 +1191,7 @@ class VimeoAlbumIE(VimeoBaseInfoExtractor):
'X-Requested-With': 'XMLHttpRequest',
})['hashed_pass']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
raise ExtractorError('Wrong password', expected=True)
raise
entries = OnDemandPagedList(functools.partial(
@@ -1309,10 +1304,10 @@ class VimeoWatchLaterIE(VimeoChannelIE): # XXX: Do not subclass from concrete I
def _page_url(self, base_url, pagenum):
url = '%s/page:%d/' % (base_url, pagenum)
- request = sanitized_Request(url)
+ request = Request(url)
# Set the header to get a partial html page with the ids,
# the normal page doesn't contain them.
- request.add_header('X-Requested-With', 'XMLHttpRequest')
+ request.headers['X-Requested-With'] = 'XMLHttpRequest'
return request
def _real_extract(self, url):
@@ -1432,7 +1427,7 @@ class VimeoProIE(VimeoBaseInfoExtractor):
**self._hidden_inputs(password_form),
}), note='Logging in with video password')
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 418:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 418:
raise ExtractorError('Wrong video password', expected=True)
raise
diff --git a/yt_dlp/extractor/vk.py b/yt_dlp/extractor/vk.py
index 575369028..6b7379d46 100644
--- a/yt_dlp/extractor/vk.py
+++ b/yt_dlp/extractor/vk.py
@@ -36,7 +36,7 @@ class VKBaseIE(InfoExtractor):
def _download_webpage_handle(self, url_or_request, video_id, *args, fatal=True, **kwargs):
response = super()._download_webpage_handle(url_or_request, video_id, *args, fatal=fatal, **kwargs)
- challenge_url, cookie = response[1].geturl() if response else '', None
+ challenge_url, cookie = response[1].url if response else '', None
if challenge_url.startswith('https://vk.com/429.html?'):
cookie = self._get_cookies(challenge_url).get('hash429')
if not cookie:
diff --git a/yt_dlp/extractor/vocaroo.py b/yt_dlp/extractor/vocaroo.py
index 704e25c22..d98fbfd2d 100644
--- a/yt_dlp/extractor/vocaroo.py
+++ b/yt_dlp/extractor/vocaroo.py
@@ -1,8 +1,6 @@
from .common import InfoExtractor
-from ..utils import (
- HEADRequest,
- float_or_none,
-)
+from ..networking import HEADRequest
+from ..utils import float_or_none
class VocarooIE(InfoExtractor):
diff --git a/yt_dlp/extractor/vodlocker.py b/yt_dlp/extractor/vodlocker.py
index 1c7236ed3..b215d6c9d 100644
--- a/yt_dlp/extractor/vodlocker.py
+++ b/yt_dlp/extractor/vodlocker.py
@@ -1,10 +1,6 @@
from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
- NO_DEFAULT,
- sanitized_Request,
- urlencode_postdata,
-)
+from ..networking import Request
+from ..utils import NO_DEFAULT, ExtractorError, urlencode_postdata
class VodlockerIE(InfoExtractor):
@@ -37,8 +33,8 @@ class VodlockerIE(InfoExtractor):
if fields['op'] == 'download1':
self._sleep(3, video_id) # they do detect when requests happen too fast!
post = urlencode_postdata(fields)
- req = sanitized_Request(url, post)
- req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ req = Request(url, post)
+ req.headers['Content-type'] = 'application/x-www-form-urlencoded'
webpage = self._download_webpage(
req, video_id, 'Downloading video page')
diff --git a/yt_dlp/extractor/voot.py b/yt_dlp/extractor/voot.py
index dd41647aa..b19a27934 100644
--- a/yt_dlp/extractor/voot.py
+++ b/yt_dlp/extractor/voot.py
@@ -1,10 +1,10 @@
import json
import time
-import urllib.error
import uuid
from .common import InfoExtractor
from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
float_or_none,
@@ -140,7 +140,7 @@ class VootIE(VootBaseIE):
'voottoken': self._TOKEN,
})['m3u8']
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
self._check_token_expiry()
raise
diff --git a/yt_dlp/extractor/vrt.py b/yt_dlp/extractor/vrt.py
index 005835712..497233d95 100644
--- a/yt_dlp/extractor/vrt.py
+++ b/yt_dlp/extractor/vrt.py
@@ -1,10 +1,10 @@
import functools
import json
import time
-import urllib.error
import urllib.parse
from .gigya import GigyaBaseIE
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
clean_html,
@@ -263,7 +263,7 @@ class VrtNUIE(VRTBaseIE):
'_csrf': self._get_cookies('https://login.vrt.be').get('OIDCXSRF').value,
}))
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
retry.error = e
continue
raise
diff --git a/yt_dlp/extractor/vrv.py b/yt_dlp/extractor/vrv.py
index ad9dc568a..523c442e6 100644
--- a/yt_dlp/extractor/vrv.py
+++ b/yt_dlp/extractor/vrv.py
@@ -8,7 +8,8 @@ import time
import urllib.parse
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_urllib_parse_urlencode
+from ..compat import compat_urllib_parse_urlencode
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
float_or_none,
@@ -54,8 +55,8 @@ class VRVBaseIE(InfoExtractor):
'?'.join([base_url, encoded_query]), video_id,
note='Downloading %s JSON metadata' % note, headers=headers, data=data)
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
- raise ExtractorError(json.loads(e.cause.read().decode())['message'], expected=True)
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+ raise ExtractorError(json.loads(e.cause.response.read().decode())['message'], expected=True)
raise
def _call_cms(self, path, video_id, note):
diff --git a/yt_dlp/extractor/weibo.py b/yt_dlp/extractor/weibo.py
index 81a23b9df..bc9a71abe 100644
--- a/yt_dlp/extractor/weibo.py
+++ b/yt_dlp/extractor/weibo.py
@@ -31,7 +31,7 @@ class WeiboIE(InfoExtractor):
# to get Referer url for genvisitor
webpage, urlh = self._download_webpage_handle(url, video_id)
- visitor_url = urlh.geturl()
+ visitor_url = urlh.url
if 'passport.weibo.com' in visitor_url:
# first visit
diff --git a/yt_dlp/extractor/weverse.py b/yt_dlp/extractor/weverse.py
index 8f2a7ee06..9a08b8e43 100644
--- a/yt_dlp/extractor/weverse.py
+++ b/yt_dlp/extractor/weverse.py
@@ -5,13 +5,13 @@ import itertools
import json
import re
import time
-import urllib.error
import urllib.parse
import uuid
from .common import InfoExtractor
from .naver import NaverBaseIE
from .youtube import YoutubeIE
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
UserNotLive,
@@ -59,7 +59,7 @@ class WeverseBaseIE(InfoExtractor):
'password': password,
}, separators=(',', ':')).encode(), headers=headers, note='Logging in')
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
raise ExtractorError('Invalid password provided', expected=True)
raise
@@ -97,10 +97,10 @@ class WeverseBaseIE(InfoExtractor):
'wmd': wmd,
})
except ExtractorError as e:
- if isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 401:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 401:
self.raise_login_required(
'Session token has expired. Log in again or refresh cookies in browser')
- elif isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+ elif isinstance(e.cause, HTTPError) and e.cause.status == 403:
raise ExtractorError('Your account does not have access to this content', expected=True)
raise
diff --git a/yt_dlp/extractor/wistia.py b/yt_dlp/extractor/wistia.py
index 884fa4b5f..bce5e8326 100644
--- a/yt_dlp/extractor/wistia.py
+++ b/yt_dlp/extractor/wistia.py
@@ -1,12 +1,12 @@
import re
-import urllib.error
import urllib.parse
from base64 import b64decode
from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
- HEADRequest,
determine_ext,
float_or_none,
int_or_none,
@@ -365,7 +365,7 @@ class WistiaChannelIE(WistiaBaseIE):
try:
data = self._download_embed_config('channel', channel_id, url)
- except (ExtractorError, urllib.error.HTTPError):
+ except (ExtractorError, HTTPError):
# Some channels give a 403 from the JSON API
self.report_warning('Failed to download channel data from API, falling back to webpage.')
webpage = self._download_webpage(f'https://fast.wistia.net/embed/channel/{channel_id}', channel_id)
diff --git a/yt_dlp/extractor/wykop.py b/yt_dlp/extractor/wykop.py
index 0fa6d524d..1d29cc89b 100644
--- a/yt_dlp/extractor/wykop.py
+++ b/yt_dlp/extractor/wykop.py
@@ -1,7 +1,7 @@
import json
-import urllib.error
from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
format_field,
@@ -43,7 +43,7 @@ class WykopBaseExtractor(InfoExtractor):
try:
return self._do_call_api(path, video_id, note, headers={'Authorization': f'Bearer {token}'})
except ExtractorError as e:
- if not retrying and isinstance(e.cause, urllib.error.HTTPError) and e.cause.code == 403:
+ if not retrying and isinstance(e.cause, HTTPError) and e.cause.status == 403:
token = self._get_token(True)
continue
raise
diff --git a/yt_dlp/extractor/xhamster.py b/yt_dlp/extractor/xhamster.py
index 7af6c8f03..37224799b 100644
--- a/yt_dlp/extractor/xhamster.py
+++ b/yt_dlp/extractor/xhamster.py
@@ -183,7 +183,7 @@ class XHamsterIE(InfoExtractor):
'height': get_height(quality),
'filesize': format_sizes.get(quality),
'http_headers': {
- 'Referer': urlh.geturl(),
+ 'Referer': urlh.url,
},
})
xplayer_sources = try_get(
diff --git a/yt_dlp/extractor/xtube.py b/yt_dlp/extractor/xtube.py
index ce4480c7d..db8292589 100644
--- a/yt_dlp/extractor/xtube.py
+++ b/yt_dlp/extractor/xtube.py
@@ -2,12 +2,12 @@ import itertools
import re
from .common import InfoExtractor
+from ..networking import Request
from ..utils import (
int_or_none,
js_to_json,
orderedSet,
parse_duration,
- sanitized_Request,
str_to_int,
url_or_none,
)
@@ -186,7 +186,7 @@ class XTubeUserIE(InfoExtractor):
entries = []
for pagenum in itertools.count(1):
- request = sanitized_Request(
+ request = Request(
'http://www.xtube.com/profile/%s/videos/%d' % (user_id, pagenum),
headers={
'Cookie': 'popunder=4',
diff --git a/yt_dlp/extractor/yesjapan.py b/yt_dlp/extractor/yesjapan.py
index b45fa8f14..94e41660d 100644
--- a/yt_dlp/extractor/yesjapan.py
+++ b/yt_dlp/extractor/yesjapan.py
@@ -1,9 +1,6 @@
from .common import InfoExtractor
-from ..utils import (
- HEADRequest,
- get_element_by_attribute,
- parse_iso8601,
-)
+from ..networking import HEADRequest
+from ..utils import get_element_by_attribute, parse_iso8601
class YesJapanIE(InfoExtractor):
@@ -42,7 +39,7 @@ class YesJapanIE(InfoExtractor):
req = self._request_webpage(
redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
if req:
- video_url = req.geturl()
+ video_url = req.url
formats = [{
'format_id': 'sd',
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 826bbb20e..2b3776aa1 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -15,13 +15,13 @@ import sys
import threading
import time
import traceback
-import urllib.error
import urllib.parse
from .common import InfoExtractor, SearchInfoExtractor
from .openload import PhantomJSwrapper
from ..compat import functools
from ..jsinterp import JSInterpreter
+from ..networking.exceptions import HTTPError, network_exceptions
from ..utils import (
NO_DEFAULT,
ExtractorError,
@@ -41,7 +41,6 @@ from ..utils import (
join_nonempty,
js_to_json,
mimetype2ext,
- network_exceptions,
orderedSet,
parse_codecs,
parse_count,
@@ -959,15 +958,15 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
except ExtractorError as e:
if not isinstance(e.cause, network_exceptions):
return self._error_or_warning(e, fatal=fatal)
- elif not isinstance(e.cause, urllib.error.HTTPError):
+ elif not isinstance(e.cause, HTTPError):
retry.error = e
continue
- first_bytes = e.cause.read(512)
+ first_bytes = e.cause.response.read(512)
if not is_html(first_bytes):
yt_error = try_get(
self._parse_json(
- self._webpage_read_content(e.cause, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
+ self._webpage_read_content(e.cause.response, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
lambda x: x['error']['message'], str)
if yt_error:
self._report_alerts([('ERROR', yt_error)], fatal=False)
@@ -975,7 +974,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
# Sometimes a 404 is also recieved. See: https://github.com/ytdl-org/youtube-dl/issues/28289
# We also want to catch all other network exceptions since errors in later pages can be troublesome
# See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
- if e.cause.code not in (403, 429):
+ if e.cause.status not in (403, 429):
retry.error = e
continue
return self._error_or_warning(e, fatal=fatal)
@@ -2837,7 +2836,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Obtain from MPD's maximum seq value
old_mpd_url = mpd_url
last_error = ctx.pop('last_error', None)
- expire_fast = immediate or last_error and isinstance(last_error, urllib.error.HTTPError) and last_error.code == 403
+ expire_fast = immediate or last_error and isinstance(last_error, HTTPError) and last_error.status == 403
mpd_url, stream_number, is_live = (mpd_feed(format_id, 5 if expire_fast else 18000)
or (mpd_url, stream_number, False))
if not refresh_sequence:
@@ -5263,7 +5262,7 @@ class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
except ExtractorError as e:
if isinstance(e.cause, network_exceptions):
- if not isinstance(e.cause, urllib.error.HTTPError) or e.cause.code not in (403, 429):
+ if not isinstance(e.cause, HTTPError) or e.cause.status not in (403, 429):
retry.error = e
continue
self._error_or_warning(e, fatal=fatal)
diff --git a/yt_dlp/extractor/zaiko.py b/yt_dlp/extractor/zaiko.py
index 84cee4445..0ccacbb6a 100644
--- a/yt_dlp/extractor/zaiko.py
+++ b/yt_dlp/extractor/zaiko.py
@@ -16,7 +16,7 @@ from ..utils import (
class ZaikoBaseIE(InfoExtractor):
def _download_real_webpage(self, url, video_id):
webpage, urlh = self._download_webpage_handle(url, video_id)
- final_url = urlh.geturl()
+ final_url = urlh.url
if 'zaiko.io/login' in final_url:
self.raise_login_required()
elif '/_buy/' in final_url:
diff --git a/yt_dlp/extractor/zattoo.py b/yt_dlp/extractor/zattoo.py
index 22620c0a3..6bd9ea064 100644
--- a/yt_dlp/extractor/zattoo.py
+++ b/yt_dlp/extractor/zattoo.py
@@ -2,7 +2,8 @@ import re
from uuid import uuid4
from .common import InfoExtractor
-from ..compat import compat_HTTPError, compat_str
+from ..compat import compat_str
+from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
@@ -36,7 +37,7 @@ class ZattooPlatformBaseIE(InfoExtractor):
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
})
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+ if isinstance(e.cause, HTTPError) and e.cause.status == 400:
raise ExtractorError(
'Unable to login: incorrect username and/or password',
expected=True)
diff --git a/yt_dlp/extractor/zype.py b/yt_dlp/extractor/zype.py
index 8cf994505..2f3b4c47f 100644
--- a/yt_dlp/extractor/zype.py
+++ b/yt_dlp/extractor/zype.py
@@ -1,7 +1,7 @@
import re
from .common import InfoExtractor
-from ..compat import compat_HTTPError
+from ..networking.exceptions import HTTPError
from ..utils import (
dict_get,
ExtractorError,
@@ -37,9 +37,9 @@ class ZypeIE(InfoExtractor):
response = self._download_json(re.sub(
r'\.(?:js|html)\?', '.json?', url), video_id)['response']
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code in (400, 401, 403):
+ if isinstance(e.cause, HTTPError) and e.cause.status in (400, 401, 403):
raise ExtractorError(self._parse_json(
- e.cause.read().decode(), video_id)['message'], expected=True)
+ e.cause.response.read().decode(), video_id)['message'], expected=True)
raise
body = response['body']
diff --git a/yt_dlp/networking/common.py b/yt_dlp/networking/common.py
index e4b362827..458eca39f 100644
--- a/yt_dlp/networking/common.py
+++ b/yt_dlp/networking/common.py
@@ -24,6 +24,7 @@ from .exceptions import (
from ..utils import (
bug_reports_message,
classproperty,
+ deprecation_warning,
error_to_str,
escape_url,
update_url_query,
@@ -507,16 +508,21 @@ class Response(io.IOBase):
# The following methods are for compatability reasons and are deprecated
@property
def code(self):
+ deprecation_warning('Response.code is deprecated, use Response.status', stacklevel=2)
return self.status
def getcode(self):
+ deprecation_warning('Response.getcode() is deprecated, use Response.status', stacklevel=2)
return self.status
def geturl(self):
+ deprecation_warning('Response.geturl() is deprecated, use Response.url', stacklevel=2)
return self.url
def info(self):
+ deprecation_warning('Response.info() is deprecated, use Response.headers', stacklevel=2)
return self.headers
def getheader(self, name, default=None):
+ deprecation_warning('Response.getheader() is deprecated, use Response.get_header', stacklevel=2)
return self.get_header(name, default)
diff --git a/yt_dlp/networking/exceptions.py b/yt_dlp/networking/exceptions.py
index 6fe8afb92..10afc9ccb 100644
--- a/yt_dlp/networking/exceptions.py
+++ b/yt_dlp/networking/exceptions.py
@@ -3,7 +3,7 @@ from __future__ import annotations
import typing
import urllib.error
-from ..utils import YoutubeDLError
+from ..utils import YoutubeDLError, deprecation_warning
if typing.TYPE_CHECKING:
from .common import RequestHandler, Response
@@ -137,6 +137,7 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
@property
def headers(self):
+ deprecation_warning('HTTPError.headers is deprecated, use HTTPError.response.headers instead')
return self._http_error.response.headers
@headers.setter
@@ -144,16 +145,20 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
return
def info(self):
+ deprecation_warning('HTTPError.info() is deprecated, use HTTPError.response.headers instead')
return self.response.headers
def getcode(self):
+ deprecation_warning('HTTPError.getcode is deprecated, use HTTPError.status instead')
return self.status
def geturl(self):
+ deprecation_warning('HTTPError.geturl is deprecated, use HTTPError.response.url instead')
return self.response.url
@property
def code(self):
+ deprecation_warning('HTTPError.code is deprecated, use HTTPError.status instead')
return self.status
@code.setter
@@ -162,6 +167,7 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
@property
def url(self):
+ deprecation_warning('HTTPError.url is deprecated, use HTTPError.response.url instead')
return self.response.url
@url.setter
@@ -170,6 +176,7 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
@property
def hdrs(self):
+ deprecation_warning('HTTPError.hdrs is deprecated, use HTTPError.response.headers instead')
return self.response.headers
@hdrs.setter
@@ -178,6 +185,7 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
@property
def filename(self):
+ deprecation_warning('HTTPError.filename is deprecated, use HTTPError.response.url instead')
return self.response.url
@filename.setter
@@ -185,6 +193,18 @@ class _CompatHTTPError(urllib.error.HTTPError, HTTPError):
return
def __getattr__(self, name):
+ # File operations are passed through the response.
+ # Warn for some commonly used ones
+ passthrough_warnings = {
+ 'read': 'response.read()',
+ # technically possibly due to passthrough, but we should discourage this
+ 'get_header': 'response.get_header()',
+ 'readable': 'response.readable()',
+ 'closed': 'response.closed',
+ 'tell': 'response.tell()',
+ }
+ if name in passthrough_warnings:
+ deprecation_warning(f'HTTPError.{name} is deprecated, use HTTPError.{passthrough_warnings[name]} instead')
return super().__getattr__(name)
def __str__(self):
diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py
index 08b0fe1ff..8cef86c43 100644
--- a/yt_dlp/postprocessor/common.py
+++ b/yt_dlp/postprocessor/common.py
@@ -1,16 +1,15 @@
import functools
import json
import os
-import urllib.error
+from ..networking import Request
+from ..networking.exceptions import HTTPError, network_exceptions
from ..utils import (
PostProcessingError,
RetryManager,
_configuration_args,
deprecation_warning,
encodeFilename,
- network_exceptions,
- sanitized_Request,
)
@@ -203,13 +202,13 @@ class PostProcessor(metaclass=PostProcessorMetaClass):
self.write_debug(f'{self.PP_NAME} query: {url}')
for retry in RetryManager(self.get_param('extractor_retries', 3), self._retry_download):
try:
- rsp = self._downloader.urlopen(sanitized_Request(url))
+ rsp = self._downloader.urlopen(Request(url))
except network_exceptions as e:
- if isinstance(e, urllib.error.HTTPError) and e.code in expected_http_errors:
+ if isinstance(e, HTTPError) and e.status in expected_http_errors:
return None
retry.error = PostProcessingError(f'Unable to communicate with {self.PP_NAME} API: {e}')
continue
- return json.loads(rsp.read().decode(rsp.info().get_param('charset') or 'utf-8'))
+ return json.loads(rsp.read().decode(rsp.headers.get_param('charset') or 'utf-8'))
class AudioConversionError(PostProcessingError): # Deprecated
diff --git a/yt_dlp/update.py b/yt_dlp/update.py
index 4790075eb..d708b09e3 100644
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@@ -7,19 +7,18 @@ import platform
import re
import subprocess
import sys
-import urllib.error
from zipimport import zipimporter
from .compat import functools # isort: split
from .compat import compat_realpath, compat_shlex_quote
+from .networking import Request
+from .networking.exceptions import HTTPError, network_exceptions
from .utils import (
Popen,
cached_method,
deprecation_warning,
- network_exceptions,
remove_end,
remove_start,
- sanitized_Request,
shell_quote,
system_identifier,
version_tuple,
@@ -190,7 +189,7 @@ class Updater:
def _get_version_info(self, tag):
url = f'{API_BASE_URL}/{self._target_repo}/releases/{tag}'
self.ydl.write_debug(f'Fetching release info: {url}')
- return json.loads(self.ydl.urlopen(sanitized_Request(url, headers={
+ return json.loads(self.ydl.urlopen(Request(url, headers={
'Accept': 'application/vnd.github+json',
'User-Agent': 'yt-dlp',
'X-GitHub-Api-Version': '2022-11-28',
@@ -315,7 +314,7 @@ class Updater:
try:
newcontent = self._download(self.release_name, self._tag)
except network_exceptions as e:
- if isinstance(e, urllib.error.HTTPError) and e.code == 404:
+ if isinstance(e, HTTPError) and e.status == 404:
return self._report_error(
f'The requested tag {self._label(self.target_channel, self.target_tag)} does not exist', True)
return self._report_network_error(f'fetch updates: {e}')
diff --git a/yt_dlp/utils/_deprecated.py b/yt_dlp/utils/_deprecated.py
index e55d42354..a8ae8ecb5 100644
--- a/yt_dlp/utils/_deprecated.py
+++ b/yt_dlp/utils/_deprecated.py
@@ -10,16 +10,6 @@ del passthrough_module
from ._utils import preferredencoding
-from ..networking._urllib import HTTPHandler
-
-# isort: split
-from .networking import random_user_agent, std_headers # noqa: F401
-from ..networking._urllib import PUTRequest # noqa: F401
-from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest # noqa: F401
-from ..networking._urllib import ProxyHandler as PerRequestProxyHandler # noqa: F401
-from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler # noqa: F401
-from ..networking._urllib import make_socks_conn_class, update_Request # noqa: F401
-from ..networking.exceptions import network_exceptions # noqa: F401
def encodeFilename(s, for_subprocess=False):
@@ -47,12 +37,3 @@ def decodeOption(optval):
def error_to_compat_str(err):
return str(err)
-
-
-class YoutubeDLHandler(HTTPHandler):
- def __init__(self, params, *args, **kwargs):
- self._params = params
- super().__init__(*args, **kwargs)
-
-
-YoutubeDLHTTPSHandler = YoutubeDLHandler
diff --git a/yt_dlp/utils/_legacy.py b/yt_dlp/utils/_legacy.py
index 96ac468b1..077000971 100644
--- a/yt_dlp/utils/_legacy.py
+++ b/yt_dlp/utils/_legacy.py
@@ -1,17 +1,30 @@
"""No longer used and new code should not use. Exists only for API compat."""
-
import platform
import struct
import sys
+import urllib.error
import urllib.parse
+import urllib.request
import zlib
from ._utils import Popen, decode_base_n, preferredencoding
from .traversal import traverse_obj
from ..dependencies import certifi, websockets
+from ..networking._helper import make_ssl_context
+from ..networking._urllib import HTTPHandler
# isort: split
+from .networking import random_user_agent, std_headers # noqa: F401
from ..cookies import YoutubeDLCookieJar # noqa: F401
+from ..networking._urllib import PUTRequest # noqa: F401
+from ..networking._urllib import SUPPORTED_ENCODINGS, HEADRequest # noqa: F401
+from ..networking._urllib import ProxyHandler as PerRequestProxyHandler # noqa: F401
+from ..networking._urllib import RedirectHandler as YoutubeDLRedirectHandler # noqa: F401
+from ..networking._urllib import ( # noqa: F401
+ make_socks_conn_class,
+ update_Request,
+)
+from ..networking.exceptions import HTTPError, network_exceptions # noqa: F401
has_certifi = bool(certifi)
has_websockets = bool(websockets)
@@ -176,5 +189,52 @@ def handle_youtubedl_headers(headers):
return filtered_headers
+def request_to_url(req):
+ if isinstance(req, urllib.request.Request):
+ return req.get_full_url()
+ else:
+ return req
+
+
+def sanitized_Request(url, *args, **kwargs):
+ from ..utils import escape_url, extract_basic_auth, sanitize_url
+ url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
+ if auth_header is not None:
+ headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
+ headers['Authorization'] = auth_header
+ return urllib.request.Request(url, *args, **kwargs)
+
+
+class YoutubeDLHandler(HTTPHandler):
+ def __init__(self, params, *args, **kwargs):
+ self._params = params
+ super().__init__(*args, **kwargs)
+
+
+YoutubeDLHTTPSHandler = YoutubeDLHandler
+
+
+class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
+ def __init__(self, cookiejar=None):
+ urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
+
+ def http_response(self, request, response):
+ return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
+
+ https_request = urllib.request.HTTPCookieProcessor.http_request
+ https_response = http_response
+
+
+def make_HTTPS_handler(params, **kwargs):
+ return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
+ verify=not params.get('nocheckcertificate'),
+ client_certificate=params.get('client_certificate'),
+ client_certificate_key=params.get('client_certificate_key'),
+ client_certificate_password=params.get('client_certificate_password'),
+ legacy_support=params.get('legacyserverconnect'),
+ use_certifi='no-certifi' not in params.get('compat_opts', []),
+ ), **kwargs)
+
+
def process_communicate_or_kill(p, *args, **kwargs):
return Popen.communicate_or_kill(p, *args, **kwargs)
diff --git a/yt_dlp/utils/_utils.py b/yt_dlp/utils/_utils.py
index d0e328716..2e619f9ea 100644
--- a/yt_dlp/utils/_utils.py
+++ b/yt_dlp/utils/_utils.py
@@ -62,11 +62,6 @@ __name__ = __name__.rsplit('.', 1)[0] # Pretend to be the parent module
compiled_regex_type = type(re.compile(''))
-USER_AGENTS = {
- 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
-}
-
-
class NO_DEFAULT:
pass
@@ -727,14 +722,6 @@ def extract_basic_auth(url):
return url, f'Basic {auth_payload.decode()}'
-def sanitized_Request(url, *args, **kwargs):
- url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
- if auth_header is not None:
- headers = args[1] if len(args) >= 2 else kwargs.setdefault('headers', {})
- headers['Authorization'] = auth_header
- return urllib.request.Request(url, *args, **kwargs)
-
-
def expand_path(s):
"""Expand shell variables and ~"""
return os.path.expandvars(compat_expanduser(s))
@@ -894,19 +881,6 @@ def formatSeconds(secs, delim=':', msec=False):
return '%s.%03d' % (ret, time.milliseconds) if msec else ret
-def make_HTTPS_handler(params, **kwargs):
- from ._deprecated import YoutubeDLHTTPSHandler
- from ..networking._helper import make_ssl_context
- return YoutubeDLHTTPSHandler(params, context=make_ssl_context(
- verify=not params.get('nocheckcertificate'),
- client_certificate=params.get('client_certificate'),
- client_certificate_key=params.get('client_certificate_key'),
- client_certificate_password=params.get('client_certificate_password'),
- legacy_support=params.get('legacyserverconnect'),
- use_certifi='no-certifi' not in params.get('compat_opts', []),
- ), **kwargs)
-
-
def bug_reports_message(before=';'):
from ..update import REPOSITORY
@@ -1143,17 +1117,6 @@ def is_path_like(f):
return isinstance(f, (str, bytes, os.PathLike))
-class YoutubeDLCookieProcessor(urllib.request.HTTPCookieProcessor):
- def __init__(self, cookiejar=None):
- urllib.request.HTTPCookieProcessor.__init__(self, cookiejar)
-
- def http_response(self, request, response):
- return urllib.request.HTTPCookieProcessor.http_response(self, request, response)
-
- https_request = urllib.request.HTTPCookieProcessor.http_request
- https_response = http_response
-
-
def extract_timezone(date_str):
m = re.search(
r'''(?x)
@@ -1455,6 +1418,7 @@ def write_string(s, out=None, encoding=None):
out.flush()
+# TODO: Use global logger
def deprecation_warning(msg, *, printer=None, stacklevel=0, **kwargs):
from .. import _IN_CLI
if _IN_CLI:
@@ -2005,13 +1969,6 @@ def url_or_none(url):
return url if re.match(r'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
-def request_to_url(req):
- if isinstance(req, urllib.request.Request):
- return req.get_full_url()
- else:
- return req
-
-
def strftime_or_none(timestamp, date_format='%Y%m%d', default=None):
datetime_object = None
try:
@@ -5525,7 +5482,7 @@ class _YDLLogger:
def warning(self, message, *, once=False):
if self._ydl:
- self._ydl.report_warning(message, only_once=once)
+ self._ydl.report_warning(message, once)
def error(self, message, *, is_error=True):
if self._ydl: