diff options
| -rw-r--r-- | youtube/util.py | 77 | ||||
| -rw-r--r-- | youtube/watch.py | 147 |
2 files changed, 110 insertions, 114 deletions
diff --git a/youtube/util.py b/youtube/util.py index b057953..30fed26 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -431,34 +431,29 @@ class RateLimitedQueue(gevent.queue.Queue): gevent.queue.Queue.__init__(self) def get(self): - self.lock.acquire() # blocks if another greenlet currently has the lock - if self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial: - gevent.sleep(self.waiting_period) - self.count_since_last_wait = 0 - - elif self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial: - self.surpassed_initial = True - gevent.sleep(self.waiting_period) - self.count_since_last_wait = 0 - - self.count_since_last_wait += 1 + with self.lock: # blocks if another greenlet currently has the lock + if ((self.count_since_last_wait >= self.subsequent_bursts and self.surpassed_initial) or + (self.count_since_last_wait >= self.initial_burst and not self.surpassed_initial)): + self.surpassed_initial = True + gevent.sleep(self.waiting_period) + self.count_since_last_wait = 0 - if not self.currently_empty and self.empty(): - self.currently_empty = True - self.empty_start = time.monotonic() + self.count_since_last_wait += 1 - item = gevent.queue.Queue.get(self) # blocks when nothing left + if not self.currently_empty and self.empty(): + self.currently_empty = True + self.empty_start = time.monotonic() - if self.currently_empty: - if time.monotonic() - self.empty_start >= self.waiting_period: - self.count_since_last_wait = 0 - self.surpassed_initial = False + item = gevent.queue.Queue.get(self) # blocks when nothing left - self.currently_empty = False + if self.currently_empty: + if time.monotonic() - self.empty_start >= self.waiting_period: + self.count_since_last_wait = 0 + self.surpassed_initial = False - self.lock.release() + self.currently_empty = False - return item + return item def download_thumbnail(save_directory, video_id): @@ -667,25 +662,6 @@ def to_valid_filename(name): # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72 INNERTUBE_CLIENTS = { - 'android_music': { - 'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI', - 'INNERTUBE_CONTEXT': { - 'client': { - 'hl': 'en', - 'gl': 'US', - 'clientName': 'ANDROID_MUSIC', - 'clientVersion': '6.48.51', - 'osName': 'Android', - 'osVersion': '14', - 'androidSdkVersion': 34, - 'platform': 'MOBILE', - 'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip' - } - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 21, - 'REQUIRE_JS_PLAYER': False - }, - 'android': { 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', 'INNERTUBE_CONTEXT': { @@ -721,6 +697,25 @@ INNERTUBE_CLIENTS = { 'REQUIRE_JS_PLAYER': False }, + 'android_music': { + 'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI', + 'INNERTUBE_CONTEXT': { + 'client': { + 'hl': 'en', + 'gl': 'US', + 'clientName': 'ANDROID_MUSIC', + 'clientVersion': '6.48.51', + 'osName': 'Android', + 'osVersion': '14', + 'androidSdkVersion': 34, + 'platform': 'MOBILE', + 'userAgent': 'com.google.android.apps.youtube.music/6.48.51 (Linux; U; Android 14; US) gzip' + } + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 21, + 'REQUIRE_JS_PLAYER': False + }, + # This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option) # See: https://github.com/zerodytrash/YouTube-Internal-Clients 'tv_embedded': { diff --git a/youtube/watch.py b/youtube/watch.py index 09a3b76..bdb23ab 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -2,6 +2,7 @@ import youtube from youtube import yt_app from youtube import util, comments, local_playlist, yt_data_extract from youtube.util import time_utc_isoformat +from youtube.util import INNERTUBE_CLIENTS import settings from flask import request @@ -369,83 +370,83 @@ def fetch_watch_page_info(video_id, playlist_id, index): return yt_data_extract.extract_watch_info_from_html(watch_page) def extract_info(video_id, use_invidious, playlist_id=None, index=None): - tasks = ( - # Get video metadata from here - gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index), - gevent.spawn(fetch_player_response, 'android', video_id) - ) - gevent.joinall(tasks) - util.check_gevent_exceptions(*tasks) - info, player_response = tasks[0].value, tasks[1].value - - yt_data_extract.update_with_new_urls(info, player_response) + for client in INNERTUBE_CLIENTS: + tasks = ( + gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index), + gevent.spawn(fetch_player_response, client, video_id) # Use client from INNERTUBE_CLIENTS + ) + gevent.joinall(tasks) + util.check_gevent_exceptions(*tasks) + info, player_response = tasks[0].value, tasks[1].value - # Age restricted video, retry - if info['age_restricted'] or info['player_urls_missing']: - if info['age_restricted']: - print('Age restricted video, retrying') - else: - print('Player urls missing, retrying') - player_response = fetch_player_response('tv_embedded', video_id) yt_data_extract.update_with_new_urls(info, player_response) - # signature decryption - decryption_error = decrypt_signatures(info, video_id) - if decryption_error: - decryption_error = 'Error decrypting url signatures: ' + decryption_error - info['playability_error'] = decryption_error - - # check if urls ready (non-live format) in former livestream - # urls not ready if all of them have no filesize - if info['was_live']: - info['urls_ready'] = False - for fmt in info['formats']: - if fmt['file_size'] is not None: - info['urls_ready'] = True - else: - info['urls_ready'] = True - - # livestream urls - # sometimes only the livestream urls work soon after the livestream is over - if (info['hls_manifest_url'] - and (info['live'] or not info['formats'] or not info['urls_ready']) - ): - manifest = util.fetch_url(info['hls_manifest_url'], - debug_name='hls_manifest.m3u8', - report_text='Fetched hls manifest' - ).decode('utf-8') - - info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest) - if not err: - info['playability_error'] = None - for fmt in info['hls_formats']: - fmt['video_quality'] = video_quality_string(fmt) - else: - info['hls_formats'] = [] - - # check for 403. Unnecessary for tor video routing b/c ip address is same - info['invidious_used'] = False - info['invidious_reload_button'] = False - info['tor_bypass_used'] = False - if (settings.route_tor == 1 - and info['formats'] and info['formats'][0]['url']): - try: - response = util.head(info['formats'][0]['url'], - report_text='Checked for URL access') - except urllib3.exceptions.HTTPError: - print('Error while checking for URL access:\n') - traceback.print_exc() - return info - - if response.status == 403: - print('Access denied (403) for video urls.') - print('Routing video through Tor') - info['tor_bypass_used'] = True + # Age restricted video, retry + if info['age_restricted'] or info['player_urls_missing']: + if info['age_restricted']: + print('Age restricted video, retrying') + else: + print('Player urls missing, retrying') + player_response = fetch_player_response('tv_embedded', video_id) + yt_data_extract.update_with_new_urls(info, player_response) + + # signature decryption + decryption_error = decrypt_signatures(info, video_id) + if decryption_error: + decryption_error = 'Error decrypting url signatures: ' + decryption_error + info['playability_error'] = decryption_error + + # check if urls ready (non-live format) in former livestream + # urls not ready if all of them have no filesize + if info['was_live']: + info['urls_ready'] = False for fmt in info['formats']: - fmt['url'] += '&use_tor=1' - elif 300 <= response.status < 400: - print('Error: exceeded max redirects while checking video URL') - return info + if fmt['file_size'] is not None: + info['urls_ready'] = True + else: + info['urls_ready'] = True + + # livestream urls + # sometimes only the livestream urls work soon after the livestream is over + if (info['hls_manifest_url'] + and (info['live'] or not info['formats'] or not info['urls_ready']) + ): + manifest = util.fetch_url(info['hls_manifest_url'], + debug_name='hls_manifest.m3u8', + report_text='Fetched hls manifest' + ).decode('utf-8') + + info['hls_formats'], err = yt_data_extract.extract_hls_formats(manifest) + if not err: + info['playability_error'] = None + for fmt in info['hls_formats']: + fmt['video_quality'] = video_quality_string(fmt) + else: + info['hls_formats'] = [] + + # check for 403. Unnecessary for tor video routing b/c ip address is same + info['invidious_used'] = False + info['invidious_reload_button'] = False + info['tor_bypass_used'] = False + if (settings.route_tor == 1 + and info['formats'] and info['formats'][0]['url']): + try: + response = util.head(info['formats'][0]['url'], + report_text='Checked for URL access') + except urllib3.exceptions.HTTPError: + print('Error while checking for URL access:\n') + traceback.print_exc() + return info + + if response.status == 403: + print('Access denied (403) for video urls.') + print('Routing video through Tor') + info['tor_bypass_used'] = True + for fmt in info['formats']: + fmt['url'] += '&use_tor=1' + elif 300 <= response.status < 400: + print('Error: exceeded max redirects while checking video URL') + return info def video_quality_string(format): |