diff options
| -rw-r--r-- | youtube/util.py | 119 | ||||
| -rw-r--r-- | youtube/watch.py | 2 |
2 files changed, 94 insertions, 27 deletions
diff --git a/youtube/util.py b/youtube/util.py index 5c0e4fb..eb121e1 100644 --- a/youtube/util.py +++ b/youtube/util.py @@ -321,7 +321,8 @@ def fetch_url(url, headers=(), timeout=15, report_text=None, data=None, response.getheader('Content-Encoding', default='identity')) if (settings.debugging_save_responses - and debug_name is not None and content): + and debug_name is not None + and content): save_dir = os.path.join(settings.data_dir, 'debug') if not os.path.exists(save_dir): os.makedirs(save_dir) @@ -394,23 +395,22 @@ def head(url, use_tor=False, report_text=None, max_redirects=10): round(time.monotonic() - start_time, 3)) return response - -mobile_user_agent = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.80 Mobile Safari/537.36' +mobile_user_agent = 'Mozilla/5.0 (Linux; Android 7.0; Redmi Note 4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36' mobile_ua = (('User-Agent', mobile_user_agent),) -desktop_user_agent = 'Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0' +desktop_user_agent = 'Mozilla/5.0 (Windows NT 6.1; rv:52.0) Gecko/20100101 Firefox/52.0' desktop_ua = (('User-Agent', desktop_user_agent),) json_header = (('Content-Type', 'application/json'),) desktop_xhr_headers = ( ('Accept', '*/*'), ('Accept-Language', 'en-US,en;q=0.5'), ('X-YouTube-Client-Name', '1'), - ('X-YouTube-Client-Version', '2.20240327.00.00'), + ('X-YouTube-Client-Version', '2.20240304.00.00'), ) + desktop_ua mobile_xhr_headers = ( ('Accept', '*/*'), ('Accept-Language', 'en-US,en;q=0.5'), - ('X-YouTube-Client-Name', '1'), - ('X-YouTube-Client-Version', '2.20240328.08.00'), + ('X-YouTube-Client-Name', '2'), + ('X-YouTube-Client-Version', '2.20240304.08.00'), ) + mobile_ua @@ -667,30 +667,47 @@ def to_valid_filename(name): # https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/extractor/youtube.py#L72 INNERTUBE_CLIENTS = { - 'web_creator': { - 'INNERTUBE_API_KEY': 'AIzaSyBUPetSUmoZL-OhlxA7wSac5XinrygCqMo', + 'android': { + 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', 'INNERTUBE_CONTEXT': { 'client': { - 'clientName': 'WEB_CREATOR', - 'clientVersion': '1.20240723.03.00', + 'hl': 'en', + 'gl': 'US', + 'clientName': 'ANDROID', + 'clientVersion': '19.09.36', + 'osName': 'Android', + 'osVersion': '12', + 'androidSdkVersion': 31, + 'platform': 'MOBILE', + 'userAgent': 'com.google.android.youtube/19.09.36 (Linux; U; Android 12; US) gzip' }, + # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287 + #'thirdParty': { + # 'embedUrl': 'https://google.com', # Can be any valid URL + #} }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 62, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 3, + 'REQUIRE_JS_PLAYER': False, }, - 'android': { + + 'android-test-suite': { 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', 'INNERTUBE_CONTEXT': { 'client': { 'hl': 'en', 'gl': 'US', - 'clientName': 'ANDROID', - 'clientVersion': '19.15.35', + 'clientName': 'ANDROID_TESTSUITE', + 'clientVersion': '1.9', 'osName': 'Android', - 'osVersion': '14', - 'androidSdkVersion': 34, + 'osVersion': '12', + 'androidSdkVersion': 31, 'platform': 'MOBILE', - 'userAgent': 'com.google.android.youtube/19.15.35 (Linux; U; Android 14; en_US; Google Pixel 6 Pro) gzip' - } + 'userAgent': 'com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip' + }, + # https://github.com/yt-dlp/yt-dlp/pull/575#issuecomment-887739287 + #'thirdParty': { + # 'embedUrl': 'https://google.com', # Can be any valid URL + #} }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 3, 'REQUIRE_JS_PLAYER': False, @@ -703,12 +720,9 @@ INNERTUBE_CLIENTS = { 'hl': 'en', 'gl': 'US', 'clientName': 'IOS', - 'clientVersion': '19.29.1', - 'deviceMake': 'Apple', - 'deviceModel': 'iPhone16,2', - 'userAgent': 'com.google.ios.youtube/19.29.1 (iPhone16,2; U; CPU iOS 17_5_1 like Mac OS X;)', - 'osName': 'iPhone', - 'osVersion': '17.5.1.21F90', + 'clientVersion': '19.09.3', + 'deviceModel': 'iPhone14,3', + 'userAgent': 'com.google.ios.youtube/19.09.3 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)' } }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, @@ -742,14 +756,62 @@ INNERTUBE_CLIENTS = { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'WEB', - 'clientVersion': '2.20240327.00.00', + 'clientVersion': '2.20220801.00.00', 'userAgent': desktop_user_agent, } }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1 }, + 'android_vr': { + 'INNERTUBE_API_KEY': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w', + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'ANDROID_VR', + 'clientVersion': '1.60.19', + 'deviceMake': 'Oculus', + 'deviceModel': 'Quest 3', + 'androidSdkVersion': 32, + 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip', + 'osName': 'Android', + 'osVersion': '12L', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 28, + 'REQUIRE_JS_PLAYER': False, + }, } +def get_visitor_data(): + visitor_data = None + visitor_data_cache = os.path.join(settings.data_dir, 'visitorData.txt') + if not os.path.exists(settings.data_dir): + os.makedirs(settings.data_dir) + if os.path.isfile(visitor_data_cache): + with open(visitor_data_cache, 'r') as file: + print('Getting visitor_data from cache') + visitor_data = file.read() + max_age = 12*3600 + file_age = time.time() - os.path.getmtime(visitor_data_cache) + if file_age > max_age: + print('visitor_data cache is too old. Removing file...') + os.remove(visitor_data_cache) + return visitor_data + + print('Fetching youtube homepage to get visitor_data') + yt_homepage = 'https://www.youtube.com' + yt_resp = fetch_url(yt_homepage, headers={'User-Agent': mobile_user_agent}, report_text='Getting youtube homepage') + visitor_data_re = r'''"visitorData":\s*?"(.+?)"''' + visitor_data_match = re.search(visitor_data_re, yt_resp.decode()) + if visitor_data_match: + visitor_data = visitor_data_match.group(1) + print(f'Got visitor_data: {len(visitor_data)}') + with open(visitor_data_cache, 'w') as file: + print('Saving visitor_data cache...') + file.write(visitor_data) + return visitor_data + else: + print('Unable to get visitor_data value') + return visitor_data def call_youtube_api(client, api, data): client_params = INNERTUBE_CLIENTS[client] @@ -757,12 +819,17 @@ def call_youtube_api(client, api, data): key = client_params['INNERTUBE_API_KEY'] host = client_params.get('INNERTUBE_HOST') or 'www.youtube.com' user_agent = context['client'].get('userAgent') or mobile_user_agent + visitor_data = get_visitor_data() url = 'https://' + host + '/youtubei/v1/' + api + '?key=' + key + if visitor_data: + context['client'].update({'visitorData': visitor_data}) data['context'] = context data = json.dumps(data) headers = (('Content-Type', 'application/json'),('User-Agent', user_agent)) + if visitor_data: + headers = ( *headers, ('X-Goog-Visitor-Id', visitor_data )) response = fetch_url( url, data=data, headers=headers, debug_name='youtubei_' + api + '_' + client, diff --git a/youtube/watch.py b/youtube/watch.py index c97d565..6c3d601 100644 --- a/youtube/watch.py +++ b/youtube/watch.py @@ -372,7 +372,7 @@ def extract_info(video_id, use_invidious, playlist_id=None, index=None): tasks = ( # Get video metadata from here gevent.spawn(fetch_watch_page_info, video_id, playlist_id, index), - gevent.spawn(fetch_player_response, 'ios', video_id) + gevent.spawn(fetch_player_response, 'android_vr', video_id) ) gevent.joinall(tasks) util.check_gevent_exceptions(*tasks) |