diff options
Diffstat (limited to 'youtube')
| -rw-r--r-- | youtube/channel.py | 2 | ||||
| -rw-r--r-- | youtube/playlist.py | 2 | ||||
| -rw-r--r-- | youtube/util.py | 115 | ||||
| -rw-r--r-- | youtube/version.py | 2 | ||||
| -rw-r--r-- | youtube/watch.py | 9 |
5 files changed, 92 insertions, 38 deletions
diff --git a/youtube/channel.py b/youtube/channel.py index b520121..81881eb 100644 --- a/youtube/channel.py +++ b/youtube/channel.py @@ -292,7 +292,7 @@ def get_number_of_videos_channel(channel_id): try: response = util.fetch_url(url, headers_mobile, debug_name='number_of_videos', report_text='Got number of videos') - except urllib.error.HTTPError as e: + except (urllib.error.HTTPError, util.FetchError) as e: traceback.print_exc() print("Couldn't retrieve number of videos") return 1000 diff --git a/youtube/playlist.py b/youtube/playlist.py index 83d530c..28b8149 100644 --- a/youtube/playlist.py +++ b/youtube/playlist.py @@ -115,7 +115,7 @@ def get_playlist_page(): video_count = yt_data_extract.deep_get(info, 'metadata', 'video_count') if video_count is None: - video_count = 40 + video_count = 1000 return flask.render_template( 'playlist.html', diff --git a/youtube/util.py b/youtube/util.py index 0416fc1..eb121e1 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -321,7 +321,8 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, response.getheader('Content-Encoding', default='identity')) if (settings.debugging_save_responses - and debug_name is not None and content): + and debug_name is not None + and content): save_dir = os.path.join(settings.data_dir, 'debug') if not os.path.exists(save_dir): os.makedirs(save_dir) @@ -394,23 +395,22 @@ def head(url, use_tor=False, report_text=None, max_redirects=10): round(time.monotonic() - start_time, 3)) return response - -mobile_user_agent = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.80 Mobile Safari/537.36' +mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36' mobile_ua = (('User-Agent', mobile_user_agent),) -desktop_user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0' +desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0' desktop_ua = (('User-Agent', desktop_user_agent),) json_header = (('Content-Type', 'application/json'),) desktop_xhr_headers = ( ('Accept', '*/*'), ('Accept-Language', 'en-US,en;q=0.5'), ('X-YouTube-Client-Name', '1'), - ('X-YouTube-Client-Version', '2.20240327.00.00'), + ('X-YouTube-Client-Version', '2.20240304.00.00'), ) + desktop_ua mobile_xhr_headers = ( ('Accept', '*/*'), ('Accept-Language', 'en-US,en;q=0.5'), - ('X-YouTube-Client-Name', '1'), - ('X-YouTube-Client-Version', '2.20240328.08.00'), + ('X-YouTube-Client-Name', '2'), + ('X-YouTube-Client-Version', '2.20240304.08.00'), ) + mobile_ua @@ -667,39 +667,47 @@ def to_valid_filename(name): # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72 INNERTUBE_CLIENTS = { - 'android_music': { - 'INNERTUBE_API_KEY': 'AIzaSyAOghZGza2MQSZkY_zfZ370N-PUdXEo8AI', + 'android': { + 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', 'INNERTUBE_CONTEXT': { 'client': { 'hl': 'en', 'gl': 'US', - 'clientName': 'ANDROID_MUSIC', - 'clientVersion': '6.44.54', + 'clientName': 'ANDROID', + 'clientVersion': '19.09.36', 'osName': 'Android', - 'osVersion': '14', - 'androidSdkVersion': 34, + 'osVersion': '12', + 'androidSdkVersion': 31, 'platform': 'MOBILE', - 'userAgent': 'com.google.android.apps.youtube.music/6.44.54 (Linux; U; Android 14; US) gzip' - } + 'userAgent': 'com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip' + }, + # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287 + #'thirdParty': { + # 'embedUrl': 'https://google.com', # Can be any valid URL + #} }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 21, - 'REQUIRE_JS_PLAYER': False + 'INNERTUBE_CONTEXT_CLIENT_NAME': 3, + 'REQUIRE_JS_PLAYER': False, }, - 'android': { + 'android-test-suite': { 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', 'INNERTUBE_CONTEXT': { 'client': { 'hl': 'en', 'gl': 'US', - 'clientName': 'ANDROID', - 'clientVersion': '19.12.36', + 'clientName': 'ANDROID_TESTSUITE', + 'clientVersion': '1.9', 'osName': 'Android', - 'osVersion': '14', - 'androidSdkVersion': 34, + 'osVersion': '12', + 'androidSdkVersion': 31, 'platform': 'MOBILE', - 'userAgent': 'com.google.android.youtube/19.13.36 (Linux; U; Android 14; en_US; Google Pixel 6 Pro) gzip' - } + 'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip' + }, + # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287 + #'thirdParty': { + # 'embedUrl': 'https://google.com', # Can be any valid URL + #} }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 3, 'REQUIRE_JS_PLAYER': False, @@ -712,9 +720,9 @@ INNERTUBE_CLIENTS = { 'hl': 'en', 'gl': 'US', 'clientName': 'IOS', - 'clientVersion': '19.12.3', + 'clientVersion': '19.09.3', 'deviceModel': 'iPhone14,3', - 'userAgent': 'com.google.ios.youtube/19.12.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)' + 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)' } }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, @@ -748,14 +756,62 @@ INNERTUBE_CLIENTS = { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'WEB', - 'clientVersion': '2.20240327.00.00', + 'clientVersion': '2.20220801.00.00', 'userAgent': desktop_user_agent, } }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1 }, + 'android_vr': { + 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'ANDROID_VR', + 'clientVersion': '1.60.19', + 'deviceMake': 'Oculus', + 'deviceModel': 'Quest 3', + 'androidSdkVersion': 32, + 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip', + 'osName': 'Android', + 'osVersion': '12L', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 28, + 'REQUIRE_JS_PLAYER': False, + }, } +def get_visitor_data(): + visitor_data = None + visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt') + if not os.path.exists(settings.data_dir): + os.makedirs(settings.data_dir) + if os.path.isfile(visitor_data_cache): + with open(visitor_data_cache, 'r') as file: + print('Getting visitor_data from cache') + visitor_data = file.read() + max_age = 12*3600 + file_age = time.time() - os.path.getmtime(visitor_data_cache) + if file_age > max_age: + print('visitor_data cache is too old. Removing file...') + os.remove(visitor_data_cache) + return visitor_data + + print('Fetching youtube homepage to get visitor_data') + yt_homepage = 'https://www.youtube.com' + yt_resp = fetch_url(yt_homepage, headers={'User-Agent': mobile_user_agent}, report_text='Getting youtube homepage') + visitor_data_re = r'''"visitorData":\s*?"(.+?)"''' + visitor_data_match = re.search(visitor_data_re, yt_resp.decode()) + if visitor_data_match: + visitor_data = visitor_data_match.group(1) + print(f'Got visitor_data: {len(visitor_data)}') + with open(visitor_data_cache, 'w') as file: + print('Saving visitor_data cache...') + file.write(visitor_data) + return visitor_data + else: + print('Unable to get visitor_data value') + return visitor_data def call_youtube_api(client, api, data): client_params = INNERTUBE_CLIENTS[client] @@ -763,12 +819,17 @@ def call_youtube_api(client, api, data): key = client_params['INNERTUBE_API_KEY'] host = client_params.get('INNERTUBE_HOST') or 'www.youtube.com' user_agent = context['client'].get('userAgent') or mobile_user_agent + visitor_data = get_visitor_data() url = 'https://' + host + '/youtubei/v1/' + api + '?key=' + key + if visitor_data: + context['client'].update({'visitorData': visitor_data}) data['context'] = context data = json.dumps(data) headers = (('Content-Type', 'application/json'),('User-Agent', user_agent)) + if visitor_data: + headers = ( *headers, ('X-Goog-Visitor-Id', visitor_data )) response = fetch_url( url, data=data, headers=headers, debug_name='youtubei_' + api + '_' + client, diff --git a/youtube/version.py b/youtube/version.py index 3679399..eaae66d 100644 --- a/youtube/version.py +++ b/youtube/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '0.2.14' +__version__ = '0.2.20' diff --git a/youtube/watch.py b/youtube/watch.py index 09a3b76..e2762cb 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -343,7 +343,6 @@ def _add_to_error(info, key, additional_message): def fetch_player_response(client, video_id): return util.call_youtube_api(client, 'player', { 'videoId': video_id, - 'params': 'CgIIAQ==', }) @@ -372,7 +371,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): tasks = ( # Get video metadata from here gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index), - gevent.spawn(fetch_player_response, 'android', video_id) + gevent.spawn(fetch_player_response, 'android_vr', video_id) ) gevent.joinall(tasks) util.check_gevent_exceptions(*tasks) @@ -650,12 +649,6 @@ def get_watch_page(video_id=None): '/videoplayback', '/videoplayback/name/' + filename) - if settings.gather_googlevideo_domains: - with open(os.path.join(settings.data_dir, 'googlevideo-domains.txt'), 'a+', encoding='utf-8') as f: - url = info['formats'][0]['url'] - subdomain = url[0:url.find(".googlevideo.com")] - f.write(subdomain + "\n") - download_formats = [] for format in (info['formats'] + info['hls_formats']): |