diff options
Diffstat (limited to 'youtube/playlist.py')
| -rw-r--r-- | youtube/playlist.py | 190 |
1 files changed, 98 insertions, 92 deletions
diff --git a/youtube/playlist.py b/youtube/playlist.py index c16dc7f..2765a30 100644 --- a/youtube/playlist.py +++ b/youtube/playlist.py @@ -1,122 +1,128 @@ +from youtube import util, yt_data_extract, proto, local_playlist +from youtube import yt_app +import settings + import base64 -import youtube.common as common import urllib import json -from string import Template -import youtube.proto as proto +import string import gevent import math - -with open("yt_playlist_template.html", "r") as file: - yt_playlist_template = Template(file.read()) - - +from flask import request, abort +import flask +def playlist_ctoken(playlist_id, offset, include_shorts=True): - -def playlist_ctoken(playlist_id, offset): - offset = proto.uint(1, offset) - # this is just obfuscation as far as I can tell. It doesn't even follow protobuf offset = b'PT:' + proto.unpadded_b64encode(offset) offset = proto.string(15, offset) + if not include_shorts: + offset += proto.string(104, proto.uint(2, 1)) + + continuation_info = proto.string(3, proto.percent_b64encode(offset)) - continuation_info = proto.string( 3, proto.percent_b64encode(offset) ) - - playlist_id = proto.string(2, 'VL' + playlist_id ) + playlist_id = proto.string(2, 'VL' + playlist_id) pointless_nest = proto.string(80226972, playlist_id + continuation_info) return base64.urlsafe_b64encode(pointless_nest).decode('ascii') -# initial request types: -# polymer_json: https://m.youtube.com/playlist?list=PLv3TTBr1W_9tppikBxAE_G6qjWdBljBHJ&pbj=1&lact=0 -# ajax json: https://m.youtube.com/playlist?list=PLv3TTBr1W_9tppikBxAE_G6qjWdBljBHJ&pbj=1&lact=0 with header X-YouTube-Client-Version: 1.20180418 +def playlist_first_page(playlist_id, report_text="Retrieved playlist", + use_mobile=False): + if use_mobile: + url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&pbj=1' + content = util.fetch_url( + url, util.mobile_xhr_headers, + report_text=report_text, debug_name='playlist_first_page' + ) + content = json.loads(content.decode('utf-8')) + else: + url = 'https://www.youtube.com/playlist?list=' + playlist_id + '&pbj=1' + content = util.fetch_url( + url, util.desktop_xhr_headers, + report_text=report_text, debug_name='playlist_first_page' + ) + content = json.loads(content.decode('utf-8')) -# continuation request types: -# polymer_json: https://m.youtube.com/playlist?&ctoken=[...]&pbj=1 -# ajax json: https://m.youtube.com/playlist?action_continuation=1&ajax=1&ctoken=[...] - + return content -headers_1 = ( - ('Accept', '*/*'), - ('Accept-Language', 'en-US,en;q=0.5'), - ('X-YouTube-Client-Name', '1'), - ('X-YouTube-Client-Version', '2.20180614'), -) -def playlist_first_page(playlist_id, report_text = "Retrieved playlist"): - url = 'https://m.youtube.com/playlist?list=' + playlist_id + '&ajax=1&disable_polymer=true' - content = common.fetch_url(url, common.mobile_ua + headers_1, report_text=report_text) - if content[0:4] == b")]}'": - content = content[4:] - content = json.loads(common.uppercase_escape(content.decode('utf-8'))) - return content - - -#https://m.youtube.com/playlist?itct=CBMQybcCIhMIptj9xJaJ2wIV2JKcCh3Idwu-&ctoken=4qmFsgI2EiRWTFBMT3kwajlBdmxWWlB0bzZJa2pLZnB1MFNjeC0tN1BHVEMaDmVnWlFWRHBEUWxFJTNE&pbj=1 -def get_videos_ajax(playlist_id, page): - - url = "https://m.youtube.com/playlist?action_continuation=1&ajax=1&ctoken=" + playlist_ctoken(playlist_id, (int(page)-1)*20) - headers = { - 'User-Agent': ' Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1', - 'Accept': '*/*', - 'Accept-Language': 'en-US,en;q=0.5', - 'X-YouTube-Client-Name': '2', - 'X-YouTube-Client-Version': '1.20180508', - } - - content = common.fetch_url(url, headers, report_text="Retrieved playlist") - '''with open('debug/playlist_debug', 'wb') as f: - f.write(content)''' - content = content[4:] - - info = json.loads(common.uppercase_escape(content.decode('utf-8'))) +def get_videos(playlist_id, page, include_shorts=True, use_mobile=False, + report_text='Retrieved playlist'): + # mobile requests return 20 videos per page + if use_mobile: + page_size = 20 + headers = util.mobile_xhr_headers + # desktop requests return 100 videos per page + else: + page_size = 100 + headers = util.desktop_xhr_headers + + url = "https://m.youtube.com/playlist?ctoken=" + url += playlist_ctoken(playlist_id, (int(page)-1)*page_size, + include_shorts=include_shorts) + url += "&pbj=1" + content = util.fetch_url( + url, headers, report_text=report_text, + debug_name='playlist_videos' + ) + + info = json.loads(content.decode('utf-8')) return info -playlist_stat_template = Template(''' -<div>$stat</div>''') -def get_playlist_page(query_string): - parameters = urllib.parse.parse_qs(query_string) - playlist_id = parameters['list'][0] - page = parameters.get("page", "1")[0] - if page == "1": +@yt_app.route('/playlist') +def get_playlist_page(): + if 'list' not in request.args: + abort(400) + + playlist_id = request.args.get('list') + page = request.args.get('page', '1') + + if page == '1': first_page_json = playlist_first_page(playlist_id) this_page_json = first_page_json else: tasks = ( - gevent.spawn(playlist_first_page, playlist_id, report_text="Retrieved playlist info" ), - gevent.spawn(get_videos_ajax, playlist_id, page) + gevent.spawn( + playlist_first_page, playlist_id, + report_text="Retrieved playlist info", use_mobile=True + ), + gevent.spawn(get_videos, playlist_id, page) ) gevent.joinall(tasks) + util.check_gevent_exceptions(*tasks) first_page_json, this_page_json = tasks[0].value, tasks[1].value - - try: - video_list = this_page_json['content']['section_list']['contents'][0]['contents'][0]['contents'] - except KeyError: - video_list = this_page_json['content']['continuation_contents']['contents'] - videos_html = '' - for video_json in video_list: - info = common.ajax_info(video_json) - videos_html += common.video_item_html(info, common.small_video_item_template) - - - metadata = common.ajax_info(first_page_json['content']['playlist_header']) - video_count = int(metadata['size'].replace(',', '')) - page_buttons = common.page_buttons_html(int(page), math.ceil(video_count/20), common.URL_ORIGIN + "/playlist", query_string) - - html_ready = common.get_html_ready(metadata) - html_ready['page_title'] = html_ready['title'] + ' - Page ' + str(page) - - stats = '' - stats += playlist_stat_template.substitute(stat=html_ready['size'] + ' videos') - stats += playlist_stat_template.substitute(stat=html_ready['views']) - return yt_playlist_template.substitute( - header = common.get_header(), - videos = videos_html, - page_buttons = page_buttons, - stats = stats, - **html_ready - )
\ No newline at end of file + + info = yt_data_extract.extract_playlist_info(this_page_json) + if info['error']: + return flask.render_template('error.html', error_message=info['error']) + + if page != '1': + info['metadata'] = yt_data_extract.extract_playlist_metadata(first_page_json) + + util.prefix_urls(info['metadata']) + for item in info.get('items', ()): + util.prefix_urls(item) + util.add_extra_html_info(item) + if 'id' in item: + item['thumbnail'] = f"{settings.img_prefix}https://i.ytimg.com/vi/{item['id']}/hq720.jpg" + + item['url'] += '&list=' + playlist_id + if item['index']: + item['url'] += '&index=' + str(item['index']) + + video_count = yt_data_extract.deep_get(info, 'metadata', 'video_count') + if video_count is None: + video_count = 1000 + + return flask.render_template( + 'playlist.html', + header_playlist_names=local_playlist.get_playlist_names(), + video_list=info.get('items', []), + num_pages=math.ceil(video_count/100), + parameters_dictionary=request.args, + + **info['metadata'] + ).encode('utf-8') |