updated from upstream | 15/02/2022 at 19:31

7 files changed, 337 insertions, 8 deletions

diff --git a/yt_dlp/extractor/atvat.py b/yt_dlp/extractor/atvat.py
index 7c30cfcbb..481a09737 100644
--- a/yt_dlp/extractor/atvat.py
+++ b/yt_dlp/extractor/atvat.py
@@ -8,6 +8,7 @@ from ..utils import (
     float_or_none,
     jwt_encode_hs256,
     try_get,
+    ExtractorError,
 )
 
 
@@ -94,6 +95,11 @@ class ATVAtIE(InfoExtractor):
             })
 
         video_id, videos_data = list(videos['data'].items())[0]
+        error_msg = try_get(videos_data, lambda x: x['error']['title'])
+        if error_msg == 'Geo check failed':
+            self.raise_geo_restricted(error_msg)
+        elif error_msg:
+            raise ExtractorError(error_msg)
         entries = [
             self._extract_video_info(url, contentResource[video['id']], video)
             for video in videos_data]
diff --git a/yt_dlp/extractor/caltrans.py b/yt_dlp/extractor/caltrans.py
new file mode 100644
index 000000000..9ac740f7e
--- /dev/null
+++ b/yt_dlp/extractor/caltrans.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class CaltransIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:[^/]+\.)?ca\.gov/vm/loc/[^/]+/(?P<id>[a-z0-9_]+)\.htm'
+    _TEST = {
+        'url': 'https://cwwp2.dot.ca.gov/vm/loc/d3/hwy50at24th.htm',
+        'info_dict': {
+            'id': 'hwy50at24th',
+            'ext': 'ts',
+            'title': 'US-50 : Sacramento : Hwy 50 at 24th',
+            'live_status': 'is_live',
+            'thumbnail': 'https://cwwp2.dot.ca.gov/data/d3/cctv/image/hwy50at24th/hwy50at24th.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        global_vars = self._search_regex(
+            r'<script[^<]+?([^<]+\.m3u8[^<]+)</script>',
+            webpage, 'Global Vars')
+        route_place = self._search_regex(r'routePlace\s*=\s*"([^"]+)"', global_vars, 'Route Place', fatal=False)
+        location_name = self._search_regex(r'locationName\s*=\s*"([^"]+)"', global_vars, 'Location Name', fatal=False)
+        poster_url = self._search_regex(r'posterURL\s*=\s*"([^"]+)"', global_vars, 'Poster Url', fatal=False)
+        video_stream = self._search_regex(r'videoStreamURL\s*=\s*"([^"]+)"', global_vars, 'Video Stream URL', fatal=False)
+
+        formats = self._extract_m3u8_formats(video_stream, video_id, 'ts', live=True)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': f'{route_place} : {location_name}',
+            'is_live': True,
+            'formats': formats,
+            'thumbnail': poster_url,
+        }
diff --git a/yt_dlp/extractor/extractors.py b/yt_dlp/extractor/extractors.py
index a9dc2289b..c3f3eb974 100644
--- a/yt_dlp/extractor/extractors.py
+++ b/yt_dlp/extractor/extractors.py
@@ -195,6 +195,7 @@ from .byutv import BYUtvIE
 from .c56 import C56IE
 from .cableav import CableAVIE
 from .callin import CallinIE
+from .caltrans import CaltransIE
 from .cam4 import CAM4IE
 from .camdemy import (
     CamdemyIE,
@@ -889,6 +890,7 @@ from .mtv import (
     MTVItaliaProgrammaIE,
 )
 from .muenchentv import MuenchenTVIE
+from .murrtube import MurrtubeIE, MurrtubeUserIE
 from .musescore import MuseScoreIE
 from .musicdex import (
     MusicdexSongIE,
@@ -1009,6 +1011,7 @@ from .niconico import (
     NicovideoSearchDateIE,
     NicovideoSearchIE,
     NicovideoSearchURLIE,
+    NicovideoTagURLIE,
 )
 from .ninecninemedia import (
     NineCNineMediaIE,
@@ -1140,6 +1143,7 @@ from .patreon import (
 )
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
+from .peekvids import PeekVidsIE, PlayVidsIE
 from .peertube import (
     PeerTubeIE,
     PeerTubePlaylistIE,
diff --git a/yt_dlp/extractor/murrtube.py b/yt_dlp/extractor/murrtube.py
new file mode 100644
index 000000000..1eb5de660
--- /dev/null
+++ b/yt_dlp/extractor/murrtube.py
@@ -0,0 +1,165 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import functools
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    OnDemandPagedList,
+    determine_ext,
+    int_or_none,
+    try_get,
+)
+
+
+class MurrtubeIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            murrtube:|
+                            https?://murrtube\.net/videos/(?P<slug>[a-z0-9\-]+)\-
+                        )
+                        (?P<id>[a-f0-9]{8}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{4}\-[a-f0-9]{12})
+                    '''
+    _TEST = {
+        'url': 'https://murrtube.net/videos/inferno-x-skyler-148b6f2a-fdcc-4902-affe-9c0f41aaaca0',
+        'md5': '169f494812d9a90914b42978e73aa690',
+        'info_dict': {
+            'id': '148b6f2a-fdcc-4902-affe-9c0f41aaaca0',
+            'ext': 'mp4',
+            'title': 'Inferno X Skyler',
+            'description': 'Humping a very good slutty sheppy (roomate)',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 284,
+            'uploader': 'Inferno Wolf',
+            'age_limit': 18,
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'tags': ['hump', 'breed', 'Fursuit', 'murrsuit', 'bareback'],
+        }
+    }
+
+    def _download_gql(self, video_id, op, note=None, fatal=True):
+        result = self._download_json(
+            'https://murrtube.net/graphql',
+            video_id, note, data=json.dumps(op).encode(), fatal=fatal,
+            headers={'Content-Type': 'application/json'})
+        return result['data']
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        data = self._download_gql(video_id, {
+            'operationName': 'Medium',
+            'variables': {
+                'id': video_id,
+            },
+            'query': '''\
+query Medium($id: ID!) {
+  medium(id: $id) {
+    title
+    description
+    key
+    duration
+    commentsCount
+    likesCount
+    viewsCount
+    thumbnailKey
+    tagList
+    user {
+      name
+      __typename
+    }
+    __typename
+  }
+}'''})
+        meta = data['medium']
+
+        storage_url = 'https://storage.murrtube.net/murrtube/'
+        format_url = storage_url + meta.get('key', '')
+        thumbnail = storage_url + meta.get('thumbnailKey', '')
+
+        if determine_ext(format_url) == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                format_url, video_id, 'mp4', entry_protocol='m3u8_native', fatal=False)
+        else:
+            formats = [{'url': format_url}]
+
+        return {
+            'id': video_id,
+            'title': meta.get('title'),
+            'description': meta.get('description'),
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': int_or_none(meta.get('duration')),
+            'uploader': try_get(meta, lambda x: x['user']['name']),
+            'view_count': meta.get('viewsCount'),
+            'like_count': meta.get('likesCount'),
+            'comment_count': meta.get('commentsCount'),
+            'tags': meta.get('tagList'),
+            'age_limit': 18,
+        }
+
+
+class MurrtubeUserIE(MurrtubeIE):
+    IE_DESC = 'Murrtube user profile'
+    _VALID_URL = r'https?://murrtube\.net/(?P<id>[^/]+)$'
+    _TEST = {
+        'url': 'https://murrtube.net/stormy',
+        'info_dict': {
+            'id': 'stormy',
+        },
+        'playlist_mincount': 27,
+    }
+    _PAGE_SIZE = 10
+
+    def _fetch_page(self, username, user_id, page):
+        data = self._download_gql(username, {
+            'operationName': 'Media',
+            'variables': {
+                'limit': self._PAGE_SIZE,
+                'offset': page * self._PAGE_SIZE,
+                'sort': 'latest',
+                'userId': user_id,
+            },
+            'query': '''\
+query Media($q: String, $sort: String, $userId: ID, $offset: Int!, $limit: Int!) {
+  media(q: $q, sort: $sort, userId: $userId, offset: $offset, limit: $limit) {
+    id
+    __typename
+  }
+}'''},
+            'Downloading page {0}'.format(page + 1))
+        if data is None:
+            raise ExtractorError(f'Failed to retrieve video list for page {page + 1}')
+
+        media = data['media']
+
+        for entry in media:
+            yield self.url_result('murrtube:{0}'.format(entry['id']), MurrtubeIE.ie_key())
+
+    def _real_extract(self, url):
+        username = self._match_id(url)
+        data = self._download_gql(username, {
+            'operationName': 'User',
+            'variables': {
+                'id': username,
+            },
+            'query': '''\
+query User($id: ID!) {
+  user(id: $id) {
+    id
+    __typename
+  }
+}'''},
+            'Downloading user info')
+        if data is None:
+            raise ExtractorError('Failed to fetch user info')
+
+        user = data['user']
+
+        entries = OnDemandPagedList(functools.partial(
+            self._fetch_page, username, user.get('id')), self._PAGE_SIZE)
+
+        return self.playlist_result(entries, username)
diff --git a/yt_dlp/extractor/niconico.py b/yt_dlp/extractor/niconico.py
index ee888e9d3..6e561bee5 100644
--- a/yt_dlp/extractor/niconico.py
+++ b/yt_dlp/extractor/niconico.py
@@ -663,6 +663,8 @@ class NiconicoPlaylistIE(InfoExtractor):
 
 
 class NicovideoSearchBaseIE(InfoExtractor):
+    _SEARCH_TYPE = 'search'
+
     def _entries(self, url, item_id, query=None, note='Downloading page %(page)s'):
         query = query or {}
         pages = [query['page']] if 'page' in query else itertools.count(1)
@@ -677,7 +679,7 @@ class NicovideoSearchBaseIE(InfoExtractor):
 
     def _search_results(self, query):
         return self._entries(
-            self._proto_relative_url(f'//www.nicovideo.jp/search/{query}'), query)
+            self._proto_relative_url(f'//www.nicovideo.jp/{self._SEARCH_TYPE}/{query}'), query)
 
 
 class NicovideoSearchIE(NicovideoSearchBaseIE, SearchInfoExtractor):
@@ -757,6 +759,25 @@ class NicovideoSearchDateIE(NicovideoSearchBaseIE, SearchInfoExtractor):
         yield from super()._entries(url, item_id, query=query, note=note)
 
 
+class NicovideoTagURLIE(NicovideoSearchBaseIE):
+    IE_NAME = 'niconico:tag'
+    IE_DESC = 'NicoNico video tag URLs'
+    _SEARCH_TYPE = 'tag'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/tag/(?P<id>[^?#&]+)?'
+    _TESTS = [{
+        'url': 'https://www.nicovideo.jp/tag/ドキュメンタリー淫夢',
+        'info_dict': {
+            'id': 'ドキュメンタリー淫夢',
+            'title': 'ドキュメンタリー淫夢'
+        },
+        'playlist_mincount': 400,
+    }]
+
+    def _real_extract(self, url):
+        query = self._match_id(url)
+        return self.playlist_result(self._entries(url, query), query, query)
+
+
 class NiconicoUserIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/user/(?P<id>\d+)/?(?:$|[#?])'
     _TEST = {
diff --git a/yt_dlp/extractor/peekvids.py b/yt_dlp/extractor/peekvids.py
new file mode 100644
index 000000000..62050a8e4
--- /dev/null
+++ b/yt_dlp/extractor/peekvids.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import remove_end
+
+
+class PeekVidsIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?peekvids\.com/
+        (?:(?:[^/?#]+/){2}|embed/?\?(?:[^#]*&)?v=)
+        (?P<id>[^/?&#]*)
+    '''
+    _TESTS = [{
+        'url': 'https://peekvids.com/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp/BSyLMbN0YCd',
+        'md5': '2ff6a357a9717dc9dc9894b51307e9a2',
+        'info_dict': {
+            'id': 'BSyLMbN0YCd',
+            'ext': 'mp4',
+            'title': 'Dane Jones - Cute redhead with perfect tits with Mini Vamp',
+            'age_limit': 18,
+        },
+    }]
+    _DOMAIN = 'www.peekvids.com'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        short_video_id = self._html_search_regex(r'<video [^>]*data-id="(.+?)"', webpage, 'short video ID')
+        srcs = self._download_json(
+            f'https://{self._DOMAIN}/v-alt/{short_video_id}', video_id,
+            note='Downloading list of source files')
+        formats = [{
+            'url': url,
+            'ext': 'mp4',
+            'format_id': name[8:],
+        } for name, url in srcs.items() if len(name) > 8 and name.startswith('data-src')]
+        if not formats:
+            formats = [{'url': url} for url in srcs.values()]
+        self._sort_formats(formats)
+
+        title = remove_end(self._html_search_regex(
+            (r'<h1.*?>\s*(.+?)\s*</h1>', r'<title>\s*(.+?)\s*</title>'),
+            webpage, 'video title', default=None), ' - PeekVids')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'age_limit': 18,
+            'formats': formats,
+        }
+
+
+class PlayVidsIE(PeekVidsIE):
+    _VALID_URL = r'https?://(?:www\.)?playvids\.com/(?:embed/|[^/]{2}/)?(?P<id>[^/?#]*)'
+    _TESTS = [{
+        'url': 'https://www.playvids.com/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
+        'md5': '2f12e50213dd65f142175da633c4564c',
+        'info_dict': {
+            'id': 'U3pBrYhsjXM',
+            'ext': 'mp4',
+            'title': 'Dane Jones - Cute redhead with perfect tits with Mini Vamp',
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'https://www.playvids.com/es/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
+        'md5': '2f12e50213dd65f142175da633c4564c',
+        'info_dict': {
+            'id': 'U3pBrYhsjXM',
+            'ext': 'mp4',
+            'title': 'Dane Jones - Cute redhead with perfect tits with Mini Vamp',
+            'age_limit': 18,
+        },
+    }, {
+        'url': 'https://www.playvids.com/embed/U3pBrYhsjXM',
+        'md5': '2f12e50213dd65f142175da633c4564c',
+        'info_dict': {
+            'id': 'U3pBrYhsjXM',
+            'ext': 'mp4',
+            'title': 'U3pBrYhsjXM',
+            'age_limit': 18,
+        },
+    }]
+    _DOMAIN = 'www.playvids.com'
diff --git a/yt_dlp/extractor/twitcasting.py b/yt_dlp/extractor/twitcasting.py
index 8c2235a8e..98ef330cb 100644
--- a/yt_dlp/extractor/twitcasting.py
+++ b/yt_dlp/extractor/twitcasting.py
@@ -86,9 +86,14 @@ class TwitCastingIE(InfoExtractor):
             request_data = urlencode_postdata({
                 'password': video_password,
             }, encoding='utf-8')
-        webpage = self._download_webpage(
+        webpage, urlh = self._download_webpage_handle(
             url, video_id, data=request_data,
             headers={'Origin': 'https://twitcasting.tv'})
+        if urlh.geturl() != url and request_data:
+            webpage = self._download_webpage(
+                urlh.geturl(), video_id, data=request_data,
+                headers={'Origin': 'https://twitcasting.tv'},
+                note='Retrying authentication')
 
         title = (clean_html(get_element_by_id('movietitle', webpage))
                  or self._html_search_meta(['og:title', 'twitter:title'], webpage, fatal=True))
@@ -149,11 +154,12 @@ class TwitCastingIE(InfoExtractor):
                 m3u8_url, video_id, ext='mp4', m3u8_id='hls',
                 live=True, headers=self._M3U8_HEADERS)
 
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, ext='mp4', m3u8_id='source',
-                live=True, query={'mode': 'source'},
-                note='Downloading source quality m3u8',
-                headers=self._M3U8_HEADERS, fatal=False))
+            if traverse_obj(stream_server_data, ('hls', 'source')):
+                formats.extend(self._extract_m3u8_formats(
+                    m3u8_url, video_id, ext='mp4', m3u8_id='source',
+                    live=True, query={'mode': 'source'},
+                    note='Downloading source quality m3u8',
+                    headers=self._M3U8_HEADERS, fatal=False))
 
             if has_websockets:
                 qq = qualities(['base', 'mobilesource', 'main'])
@@ -164,11 +170,12 @@ class TwitCastingIE(InfoExtractor):
                         'format_id': 'ws-%s' % mode,
                         'ext': 'mp4',
                         'quality': qq(mode),
+                        'source_preference': -10,
                         # TwitCasting simply sends moof atom directly over WS
                         'protocol': 'websocket_frag',
                     })
 
-            self._sort_formats(formats)
+            self._sort_formats(formats, ('source',))
 
             infodict = {
                 'formats': formats